Merge #4624

4624: Add "precommands" to benchmark r=dureuill a=dureuill # Pull Request ## Related issue Helps for #4493 ## What does this PR do? - Add support for precommands for cargo xtask bench - update benchmark docs - update workload files Co-authored-by: Louis Dureuil <louis@meilisearch.com>
meilisearch · May 13, 2024 · 76bb6d5 · 76bb6d5
2 parents 95fcd17 + 9d3ff11
commit 76bb6d5
Show file tree

Hide file tree

Showing 10 changed files with 71 additions and 24 deletions.
diff --git a/BENCHMARKS.md b/BENCHMARKS.md
@@ -187,8 +187,8 @@ They are JSON files with the following structure (comments are not actually supp
   },
   // Core of the workload.
   // A list of commands to run sequentially.
-  // A command is a request to the Meilisearch instance that is executed while the profiling runs.
-  "commands": [
+  // Optional: A precommand is a request to the Meilisearch instance that is executed before the profiling runs.
+  "precommands": [
     {
       // Meilisearch route to call. `http://localhost:7700/` will be prepended.
       "route": "indexes/movies/settings",
@@ -224,8 +224,11 @@ They are JSON files with the following structure (comments are not actually supp
       // - DontWait: run the next command without waiting the response to this one.
       // - WaitForResponse: run the next command as soon as the response from the server is received.
       // - WaitForTask: run the next command once **all** the Meilisearch tasks created up to now have finished processing.
-      "synchronous": "DontWait"
-    },
+      "synchronous": "WaitForTask"
+    }
+  ],
+  // A command is a request to the Meilisearch instance that is executed while the profiling runs.
+  "commands": [
     {
       "route": "indexes/movies/documents",
       "method": "POST",

diff --git a/workloads/hackernews.json b/workloads/hackernews.json
@@ -54,7 +54,7 @@
       "sha256": "27e25efd0b68b159b8b21350d9af76938710cb29ce0393fa71b41c4f3c630ffe"
     }
   },
-  "commands": [
+  "precommands": [
     {
       "route": "indexes/movies/settings",
       "method": "PATCH",
@@ -78,8 +78,10 @@
           ]
         }
       },
-      "synchronous": "DontWait"
-    },
+      "synchronous": "WaitForTask"
+    }
+  ],
+  "commands": [
     {
       "route": "indexes/movies/documents",
       "method": "POST",

diff --git a/workloads/movies-nothreads.json b/workloads/movies-nothreads.json
@@ -11,7 +11,7 @@
       "sha256": "5b6e4cb660bc20327776e8a33ea197b43d9ec84856710ead1cc87ab24df77de1"
     }
   },
-  "commands": [
+  "precommands": [
     {
       "route": "indexes/movies/settings",
       "method": "PATCH",
@@ -30,8 +30,10 @@
           ]
         }
       },
-      "synchronous": "DontWait"
-    },
+      "synchronous": "WaitForTask"
+    }
+  ],
+  "commands": [
     {
       "route": "indexes/movies/documents",
       "method": "POST",

diff --git a/workloads/movies-subset-hf-embeddings.json b/workloads/movies-subset-hf-embeddings.json
@@ -11,7 +11,7 @@
       "sha256": "d215e395e4240f12f03b8f1f68901eac82d9e7ded5b462cbf4a6b8efde76c6c6"
     }
   },
-  "commands": [
+  "precommands": [
     {
       "route": "experimental-features",
       "method": "PATCH",
@@ -55,7 +55,9 @@
         }
       },
       "synchronous": "WaitForTask"
-    },
+    }
+  ],
+  "commands": [
     {
       "route": "indexes/movies/documents",
       "method": "POST",

diff --git a/workloads/settings-add-embeddings.json b/workloads/settings-add-embeddings.json
@@ -11,7 +11,7 @@
       "sha256": "d215e395e4240f12f03b8f1f68901eac82d9e7ded5b462cbf4a6b8efde76c6c6"
     }
   },
-  "commands": [
+  "precommands": [
     {
       "route": "experimental-features",
       "method": "PATCH",
@@ -49,7 +49,9 @@
         "asset": "movies-100.json"
       },
       "synchronous": "WaitForTask"
-    },
+    }
+  ],
+  "commands": [
     {
       "route": "indexes/movies/settings",
       "method": "PATCH",

diff --git a/workloads/settings-add-remove-filters.json b/workloads/settings-add-remove-filters.json
@@ -11,7 +11,7 @@
       "sha256": "28c359a0956958af0ba204ec11bad3045a0864a10b4838914fea25a01724f84b"
     }
   },
-  "commands": [
+  "precommands": [
     {
       "route": "indexes/peoples/settings",
       "method": "PATCH",
@@ -59,7 +59,9 @@
         "asset": "150k-people.json"
       },
       "synchronous": "WaitForTask"
-    },
+    }
+  ],
+  "commands": [
     {
       "route": "indexes/peoples/settings",
       "method": "PATCH",

diff --git a/workloads/settings-proximity-precision.json b/workloads/settings-proximity-precision.json
@@ -11,7 +11,7 @@
       "sha256": "28c359a0956958af0ba204ec11bad3045a0864a10b4838914fea25a01724f84b"
     }
   },
-  "commands": [
+  "precommands": [
     {
       "route": "indexes/peoples/settings",
       "method": "PATCH",
@@ -61,7 +61,9 @@
         "asset": "150k-people.json"
       },
       "synchronous": "WaitForTask"
-    },
+    }
+  ],
+  "commands": [
     {
       "route": "indexes/peoples/settings",
       "method": "PATCH",

diff --git a/workloads/settings-remove-add-swap-searchable.json b/workloads/settings-remove-add-swap-searchable.json
@@ -11,7 +11,7 @@
       "sha256": "28c359a0956958af0ba204ec11bad3045a0864a10b4838914fea25a01724f84b"
     }
   },
-  "commands": [
+  "precommands": [
     {
       "route": "indexes/peoples/settings",
       "method": "PATCH",
@@ -61,7 +61,9 @@
         "asset": "150k-people.json"
       },
       "synchronous": "WaitForTask"
-    },
+    }
+  ],
+  "commands": [
     {
       "route": "indexes/peoples/settings",
       "method": "PATCH",

diff --git a/workloads/settings-typo.json b/workloads/settings-typo.json
@@ -11,7 +11,7 @@
       "sha256": "28c359a0956958af0ba204ec11bad3045a0864a10b4838914fea25a01724f84b"
     }
   },
-  "commands": [
+  "precommands": [
     {
       "route": "indexes/peoples/settings",
       "method": "PATCH",
@@ -62,14 +62,18 @@
         "asset": "150k-people.json"
       },
       "synchronous": "WaitForTask"
-    },
+    }
+  ],
+  "commands": [
     {
       "route": "indexes/peoples/settings",
       "method": "PATCH",
       "body": {
         "inline": {
           "typoTolerance": {
-            "disableOnAttributes": ["featured_job_organization_name"]
+            "disableOnAttributes": [
+              "featured_job_organization_name"
+            ]
           }
         }
       },
@@ -93,7 +97,22 @@
       "body": {
         "inline": {
           "typoTolerance": {
-            "disableOnWords": ["Ben","Elowitz","Kevin","Flaherty", "Ron", "Dustin", "Owen", "Chris", "Mark", "Matt", "Peter", "Van", "Head", "of"]
+            "disableOnWords": [
+              "Ben",
+              "Elowitz",
+              "Kevin",
+              "Flaherty",
+              "Ron",
+              "Dustin",
+              "Owen",
+              "Chris",
+              "Mark",
+              "Matt",
+              "Peter",
+              "Van",
+              "Head",
+              "of"
+            ]
           }
         }
       },

diff --git a/xtask/src/bench/workload.rs b/xtask/src/bench/workload.rs
@@ -22,6 +22,8 @@ pub struct Workload {
     pub run_count: u16,
     pub extra_cli_args: Vec<String>,
     pub assets: BTreeMap<String, Asset>,
+    #[serde(default)]
+    pub precommands: Vec<super::command::Command>,
     pub commands: Vec<super::command::Command>,
 }
 
@@ -37,6 +39,15 @@ async fn run_commands(
     let report_folder = &args.report_folder;
     let workload_name = &workload.name;
 
+    for batch in workload
+        .precommands
+        .as_slice()
+        .split_inclusive(|command| !matches!(command.synchronous, SyncMode::DontWait))
+    {
+        super::command::run_batch(meili_client, batch, &workload.assets, &args.asset_folder)
+            .await?;
+    }
+
     std::fs::create_dir_all(report_folder)
         .with_context(|| format!("could not create report directory at {report_folder}"))?;