Tag runs as lukewarm

README.md

@@ -114,11 +114,19 @@ If a system is of a "multidimensional OLAP" kind, and so is always or implicitly
 
 ### Caching
 
-If the system contains a cache for query results, it should be disabled.
+We distinguish three cases:
 
-It is okay if the system performs caching for source data (buffer pools and similar). If the cache or buffer pools can be flushed, they should be flushed before the first run of every query.
+1. Hot runs. The database starts and then runs all queries sequentially, with three runs per query.
 
-If the system contains a cache for intermediate data, that cache should be disabled if it is located near the end of the query execution pipeline, thus similar to a query result cache.
+2. Cold runs. Before each of the three runs per query, all caches are cleared. This means a) clearing the operating system page cache and b) all caches within the database, including buffer pools. Many databases provide commands to clear internal caches. However, for fairness towards systems which don't offer such statements, it is required to _restart_ the database.
+
+3. Lukewarm runs. Similar to cold runs but _only_ the operating system page cache is cleared before each query. This is what ClickBench historically considered as "cold run", benefiting databases with aggressive internal caching. The benchmark is migrating from lukewarm runs to true cold runs (previous case 2.). Submissions that do not properly restart the database-under-test have a tag "lukewarm-cold-run". Please change the script to a true cold run, remove the label, and send a PR against the repository - thanks.
+
+General rules regarding caching:
+- Query result caches should be disabled.
+- Caching source data (e.g. buffer pools) is fine.
+- Caches for intermediate data (e.g. hash tables) are generally okay, however if such caches are located near the end of the query
+  execution pipeline, the effects are similar to query result caching and such caches should thus be disabled.
 
 ### Incomplete Results
 

arc/results/c6a.2xlarge.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["Python", "column-oriented", "time-series"],
+    "tags": ["Python", "column-oriented", "time-series", "lukewarm-cold-run"],
     "load_time": 0,
     "data_size": 14779976446,
     "result": [

arc/results/c6a.4xlarge.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["Python", "column-oriented", "time-series"],
+    "tags": ["Python", "column-oriented", "time-series", "lukewarm-cold-run"],
     "load_time": 0,
     "data_size": 14779976446,
     "result": [

arc/results/c6a.metal.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["Python", "column-oriented", "time-series"],
+    "tags": ["Python", "column-oriented", "time-series", "lukewarm-cold-run"],
     "load_time": 0,
     "data_size": 14779976446,
     "result": [

arc/results/c8g.metal-48xl.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["Python", "column-oriented", "time-series"],
+    "tags": ["Python", "column-oriented", "time-series", "lukewarm-cold-run"],
     "load_time": 0,
     "data_size": 14779976446,
     "result": [

arc/template.json

@@ -5,6 +5,7 @@
   "tags": [
     "Python",
     "column-oriented",
-    "time-series"
+    "time-series",
+    "lukewarm-cold-run"
   ]
 }

byconity/results/c6a.4xlarge.json

@@ -7,7 +7,7 @@
     "tuned": "no",
     "comment": "",
 
-    "tags": ["C++", "column-oriented", "ClickHouse derivative"],
+    "tags": ["C++", "column-oriented", "ClickHouse derivative", "lukewarm-cold-run"],
 
     "load_time": 343.482,
     "data_size": 14602455235,

byconity/template.json

@@ -2,5 +2,5 @@
     "system": "ByConity",
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++", "column-oriented", "ClickHouse derivative"]
+    "tags": ["C++", "column-oriented", "ClickHouse derivative","lukewarm-cold-run"]
 }

chdb-dataframe/results/c6a.metal.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless","Python","dataframe"],
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless","Python","dataframe", "lukewarm-cold-run"],
     "load_time": 101,
     "data_size": 155852128256,
     "result": [

chdb-dataframe/results/c7a.metal-48xl.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless","Python","dataframe"],
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless","Python","dataframe", "lukewarm-cold-run"],
     "load_time": 78,
     "data_size": 190650941440,
     "result": [

chdb-dataframe/results/c7i.metal-48xl.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless","Python","dataframe"],
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless","Python","dataframe", "lukewarm-cold-run"],
     "load_time": 140,
     "data_size": 153826500608,
     "result": [

chdb-dataframe/results/c8g.metal-48xl.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless","Python","dataframe"],
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless","Python","dataframe", "lukewarm-cold-run"],
     "load_time": 72,
     "data_size": 196759048192,
     "result": [

chdb-dataframe/template.json

@@ -2,5 +2,5 @@
     "system": "chDB (DataFrame)",
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless","Python","dataframe"]
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless","Python","dataframe","lukewarm-cold-run"]
 }

chdb-parquet/results/c6a.2xlarge.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
     "load_time": 0,
     "data_size": 14737666736,
     "result": [

chdb-parquet/results/c6a.4xlarge.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
     "load_time": 0,
     "data_size": 14737666736,
     "result": [

chdb-parquet/results/c6a.large.json

@@ -5,7 +5,7 @@
   "cluster_size": 1,
   "proprietary": "no",
   "tuned": "no",
-  "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+  "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
   "load_time": 0,
   "data_size": 14737666736,
   "result": [

chdb-parquet/results/c6a.metal.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
     "load_time": 0,
     "data_size": 14737666736,
     "result": [

chdb-parquet/results/c6a.xlarge.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
     "load_time": 0,
     "data_size": 14737666736,
     "result": [

chdb-parquet/results/c7a.metal-48xl.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary":  "no",
     "tuned":  "no",
-    "tags":  ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+    "tags":  ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
     "load_time": 0,
     "data_size": 14737666736,
     "result": [

chdb-parquet/results/c7i.metal-48xl.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary":  "no",
     "tuned":  "no",
-    "tags":  ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+    "tags":  ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
     "load_time": 0,
     "data_size": 14737666736,
     "result": [

chdb-parquet/results/c8g.4xlarge.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary":   "no",
     "tuned":   "no",
-    "tags":   ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+    "tags":   ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
     "load_time": 0,
     "data_size": 14737666736,
     "result": [

chdb-parquet/results/t3a.small.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
     "load_time": 0,
     "data_size": 14737666736,
     "result": [

chdb-parquet/template.json

@@ -2,5 +2,5 @@
     "system": "chDB (Parquet, partitioned)",
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"]
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless","lukewarm-cold-run"]
 }

chdb/results/c6a.2xlarge.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
     "load_time": 561,
     "data_size": 14477033643,
     "result": [

chdb/results/c6a.4xlarge.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
     "load_time": 516,
     "data_size": 14477072504,
     "result": [

chdb/results/c6a.large.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
     "load_time": 818,
     "data_size": 14467595822,
     "result": [

chdb/results/c6a.metal.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
     "load_time": 134,
     "data_size": 14469728876,
     "result": [

chdb/results/c6a.xlarge.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
     "load_time": 608,
     "data_size": 14465782664,
     "result": [

chdb/results/c7a.metal-48xl.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary":  "no",
     "tuned":  "no",
-    "tags":  ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+    "tags":  ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
     "load_time": 121,
     "data_size": 14470455838,
     "result": [

chdb/results/c7i.metal-48xl.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary":  "no",
     "tuned":  "no",
-    "tags":  ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+    "tags":  ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
     "load_time": 449,
     "data_size": 14477033643,
     "result": [

chdb/results/c8g.4xlarge.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary":   "no",
     "tuned":   "no",
-    "tags":   ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+    "tags":   ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
     "load_time": 0.063,
     "data_size": 14470258174,
     "result": [

chdb/results/t3a.small.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"],
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless", "lukewarm-cold-run"],
     "load_time": 1490,
     "data_size": 14464160379,
     "result": [

chdb/template.json

@@ -2,5 +2,5 @@
     "system": "chDB",
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless"]
+    "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless","serverless","lukewarm-cold-run"]
 }

chyt/results/yt.192GB_YC.json

@@ -8,7 +8,7 @@
     "tuned": "no",
     "comment": "",
 
-    "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed"],
+    "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed", "lukewarm-cold-run"],
 
     "load_time": 0,
     "data_size": 11991598975,

citus/results/c6a.2xlarge.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary":   "no",
     "tuned":   "no",
-    "tags":   ["C","PostgreSQL compatible","column-oriented"],
+    "tags":   ["C","PostgreSQL compatible","column-oriented", "lukewarm-cold-run"],
     "load_time": 1439,
     "data_size": 18980595583,
     "result": [
@@ -53,4 +53,4 @@
         [1.314, 0.591, 0.59],
         [1.414, 0.682, 0.675]
 ]
-}
+}

citus/results/c6a.4xlarge.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary":  "no",
     "tuned":  "no",
-    "tags":  ["C","PostgreSQL compatible","column-oriented"],
+    "tags":  ["C","PostgreSQL compatible","column-oriented", "lukewarm-cold-run"],
     "load_time": 1408,
     "data_size": 18980595583,
     "result": [
@@ -53,4 +53,4 @@
         [1.39, 0.584, 0.584],
         [1.417, 0.67, 0.656]
 ]
-}
+}

citus/results/c6a.large.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary":   "no",
     "tuned":   "no",
-    "tags":   ["C","PostgreSQL compatible","column-oriented"],
+    "tags":   ["C","PostgreSQL compatible","column-oriented", "lukewarm-cold-run"],
     "load_time": 1714,
     "data_size": 18980595587,
     "result": [

citus/results/c6a.xlarge.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary":   "no",
     "tuned":   "no",
-    "tags":   ["C","PostgreSQL compatible","column-oriented"],
+    "tags":   ["C","PostgreSQL compatible","column-oriented", "lukewarm-cold-run"],
     "load_time": 1596,
     "data_size": 18980595587,
     "result": [

citus/template.json

@@ -2,5 +2,5 @@
     "system": "Citus",
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C","PostgreSQL compatible","column-oriented"]
+    "tags": ["C","PostgreSQL compatible","column-oriented","lukewarm-cold-run"]
 }

clickhouse-datalake-partitioned/results/c6a.2xlarge.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"],
+    "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative", "lukewarm-cold-run"],
     "load_time": 0,
     "data_size": 14737666736,
     "result": [

clickhouse-datalake-partitioned/results/c6a.4xlarge.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"],
+    "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative", "lukewarm-cold-run"],
     "load_time": 0,
     "data_size": 14737666736,
     "result": [

clickhouse-datalake-partitioned/results/c6a.large.json

@@ -5,7 +5,7 @@
     "cluster_size": 1,
     "proprietary": "no",
     "tuned": "no",
-    "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"],
+    "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative", "lukewarm-cold-run"],
     "load_time": 0,
     "data_size": 14737666736,
     "result": [