From 1bf33c1ce1546b6f61dba0d27f7eae449d8d841d Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Tue, 6 Feb 2024 10:01:34 -0500 Subject: [PATCH 1/8] Extending ingestion benchmarks. Included a 10M docs single field benchmark --- tests/benchmarks/json_set_10M_docs_single_text.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/benchmarks/json_set_10M_docs_single_text.yml diff --git a/tests/benchmarks/json_set_10M_docs_single_text.yml b/tests/benchmarks/json_set_10M_docs_single_text.yml new file mode 100644 index 000000000..d8262e126 --- /dev/null +++ b/tests/benchmarks/json_set_10M_docs_single_text.yml @@ -0,0 +1,11 @@ +version: 0.2 +name: "json_set_10M_docs_single_text" +description: "JSON.SET of 10M docs with a single text field" + + +clientconfig: + - tool: ftsb_redisearch + - parameters: + - workers: 64 + - reporting-period: 1s + - input: "https://s3.amazonaws.com/benchmarks.redislabs/redisearch/datasets/10M-singlevalue-text-json/10M-singlevalue-text-json.redisjson.commands.SETUP.csv" From ed1dc666e6383a6eb5df82c516fb0622dc777237 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Tue, 6 Feb 2024 12:12:31 -0500 Subject: [PATCH 2/8] Extending ingestion benchmarks. Included a 80M docs single field benchmark --- tests/benchmarks/json_set_80M_docs_single_text.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/benchmarks/json_set_80M_docs_single_text.yml diff --git a/tests/benchmarks/json_set_80M_docs_single_text.yml b/tests/benchmarks/json_set_80M_docs_single_text.yml new file mode 100644 index 000000000..3572fe66b --- /dev/null +++ b/tests/benchmarks/json_set_80M_docs_single_text.yml @@ -0,0 +1,11 @@ +version: 0.2 +name: "json_set_80M_docs_single_text" +description: "JSON.SET of 80M docs with a single text field" + + +clientconfig: + - tool: ftsb_redisearch + - parameters: + - workers: 64 + - reporting-period: 1s + - input: "https://s3.amazonaws.com/benchmarks.redislabs/redisearch/datasets/80M-singlevalue-text-json/80M-singlevalue-text-json.redisjson.commands.SETUP.csv" From f5f8df9cd3cf6dab06f89ba86bce19cba2622b9a Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Tue, 6 Feb 2024 15:25:13 -0500 Subject: [PATCH 3/8] Running benchmarks faster by spliting from 3 to 5 envs --- .github/workflows/benchmark-runner.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-runner.yml b/.github/workflows/benchmark-runner.yml index fb1dd6c2b..8577dc276 100644 --- a/.github/workflows/benchmark-runner.yml +++ b/.github/workflows/benchmark-runner.yml @@ -17,7 +17,7 @@ jobs: benchmark-json-oss-standalone: strategy: matrix: - member_id: [1, 2, 3] + member_id: [1, 2, 3, 4, 5] uses: ./.github/workflows/benchmark-flow.yml secrets: inherit with: @@ -27,7 +27,7 @@ jobs: benchmark-json-oss-standalone-profiler: strategy: matrix: - member_id: [1, 2, 3] + member_id: [1, 2, 3, 4, 5] uses: ./.github/workflows/benchmark-flow.yml secrets: inherit with: From b301a45f777ac1d5a428389f5541d7bb4d727d5a Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Mon, 19 Feb 2024 17:29:30 +0000 Subject: [PATCH 4/8] Added a 10M docs array single field benchmark --- tests/benchmarks/.gitignore | 4 ++++ .../json_set_10M_docs_single_text_array.yml | 11 +++++++++++ tests/benchmarks/json_set_80M_docs_single_text.yml | 11 ----------- 3 files changed, 15 insertions(+), 11 deletions(-) create mode 100644 tests/benchmarks/json_set_10M_docs_single_text_array.yml delete mode 100644 tests/benchmarks/json_set_80M_docs_single_text.yml diff --git a/tests/benchmarks/.gitignore b/tests/benchmarks/.gitignore index afe427269..57686ad02 100644 --- a/tests/benchmarks/.gitignore +++ b/tests/benchmarks/.gitignore @@ -4,3 +4,7 @@ datasets/*.rdb *.rdb binaries +*.rdb +*.log +*.zip + diff --git a/tests/benchmarks/json_set_10M_docs_single_text_array.yml b/tests/benchmarks/json_set_10M_docs_single_text_array.yml new file mode 100644 index 000000000..155000e94 --- /dev/null +++ b/tests/benchmarks/json_set_10M_docs_single_text_array.yml @@ -0,0 +1,11 @@ +version: 0.2 +name: "json_set_10M_docs_single_text_array" +description: "JSON.SET of 10M docs with a single text field whitin an array" + + +clientconfig: + - tool: ftsb_redisearch + - parameters: + - workers: 64 + - reporting-period: 1s + - input: "https://s3.amazonaws.com/benchmarks.redislabs/redisearch/datasets/10M-singlevalue-text-json-array/10M-singlevalue-text-json-array.redisjson.commands.SETUP.csv" diff --git a/tests/benchmarks/json_set_80M_docs_single_text.yml b/tests/benchmarks/json_set_80M_docs_single_text.yml deleted file mode 100644 index 3572fe66b..000000000 --- a/tests/benchmarks/json_set_80M_docs_single_text.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 0.2 -name: "json_set_80M_docs_single_text" -description: "JSON.SET of 80M docs with a single text field" - - -clientconfig: - - tool: ftsb_redisearch - - parameters: - - workers: 64 - - reporting-period: 1s - - input: "https://s3.amazonaws.com/benchmarks.redislabs/redisearch/datasets/80M-singlevalue-text-json/80M-singlevalue-text-json.redisjson.commands.SETUP.csv" From 2f46922826b9dee5c264bae4280cc1b49a58f74e Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Wed, 10 Apr 2024 18:11:16 +0100 Subject: [PATCH 5/8] Added bool, numeric float, and int single field benchmarks --- tests/benchmarks/json_set_10M_docs_single_bool.yml | 11 +++++++++++ .../json_set_10M_docs_single_numeric_float.yml | 11 +++++++++++ .../json_set_10M_docs_single_numeric_int.yml | 11 +++++++++++ 3 files changed, 33 insertions(+) create mode 100644 tests/benchmarks/json_set_10M_docs_single_bool.yml create mode 100644 tests/benchmarks/json_set_10M_docs_single_numeric_float.yml create mode 100644 tests/benchmarks/json_set_10M_docs_single_numeric_int.yml diff --git a/tests/benchmarks/json_set_10M_docs_single_bool.yml b/tests/benchmarks/json_set_10M_docs_single_bool.yml new file mode 100644 index 000000000..ad53f6fa7 --- /dev/null +++ b/tests/benchmarks/json_set_10M_docs_single_bool.yml @@ -0,0 +1,11 @@ +version: 0.2 +name: "json_set_10M_docs_single_bool" +description: "JSON.SET of 10M docs with a single bool field" + + +clientconfig: + - tool: ftsb_redisearch + - parameters: + - workers: 64 + - reporting-period: 1s + - input: "//s3.amazonaws.com/benchmarks.redislabs/redisearch/datasets/10M-singlevalue-bool-json/10M-singlevalue-bool-json.redisjson.commands.SETUP.csv" diff --git a/tests/benchmarks/json_set_10M_docs_single_numeric_float.yml b/tests/benchmarks/json_set_10M_docs_single_numeric_float.yml new file mode 100644 index 000000000..674d8d0bc --- /dev/null +++ b/tests/benchmarks/json_set_10M_docs_single_numeric_float.yml @@ -0,0 +1,11 @@ +version: 0.2 +name: "json_set_10M_docs_single_numeric_float" +description: "JSON.SET of 10M docs with a single numeric field with float values" + + +clientconfig: + - tool: ftsb_redisearch + - parameters: + - workers: 64 + - reporting-period: 1s + - input: "https://s3.amazonaws.com/benchmarks.redislabs/redisearch/datasets/10M-singlevalue-numeric-float-json/10M-singlevalue-numeric-float-json.redisjson.commands.SETUP.csv" diff --git a/tests/benchmarks/json_set_10M_docs_single_numeric_int.yml b/tests/benchmarks/json_set_10M_docs_single_numeric_int.yml new file mode 100644 index 000000000..296abf9e4 --- /dev/null +++ b/tests/benchmarks/json_set_10M_docs_single_numeric_int.yml @@ -0,0 +1,11 @@ +version: 0.2 +name: "json_set_10M_docs_single_numeric_int" +description: "JSON.SET of 10M docs with a single numeric field with int values" + + +clientconfig: + - tool: ftsb_redisearch + - parameters: + - workers: 64 + - reporting-period: 1s + - input: "https://s3.amazonaws.com/benchmarks.redislabs/redisearch/datasets/10M-singlevalue-numeric-int-json/10M-singlevalue-numeric-int-json.redisjson.commands.SETUP.csv" From 996b04ad1bbdc39eebc14c907d0b5a8e7bf469ce Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Wed, 24 Apr 2024 00:14:46 +0100 Subject: [PATCH 6/8] Included 5 new array ingestion benchmarks --- .../json_set_10M_docs_array_nested_array_text.yml | 11 +++++++++++ ...M_docs_array_nested_dict_with_array_with_text.yml | 11 +++++++++++ ...t_10M_docs_array_nested_dict_with_numeric_int.yml | 11 +++++++++++ ...json_set_10M_docs_array_nested_dict_with_text.yml | 11 +++++++++++ .../json_set_10M_docs_array_single_numeric_int.yml | 12 ++++++++++++ ...y.yml => json_set_10M_docs_array_single_text.yml} | 2 +- 6 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 tests/benchmarks/json_set_10M_docs_array_nested_array_text.yml create mode 100644 tests/benchmarks/json_set_10M_docs_array_nested_dict_with_array_with_text.yml create mode 100644 tests/benchmarks/json_set_10M_docs_array_nested_dict_with_numeric_int.yml create mode 100644 tests/benchmarks/json_set_10M_docs_array_nested_dict_with_text.yml create mode 100644 tests/benchmarks/json_set_10M_docs_array_single_numeric_int.yml rename tests/benchmarks/{json_set_10M_docs_single_text_array.yml => json_set_10M_docs_array_single_text.yml} (89%) diff --git a/tests/benchmarks/json_set_10M_docs_array_nested_array_text.yml b/tests/benchmarks/json_set_10M_docs_array_nested_array_text.yml new file mode 100644 index 000000000..281553c48 --- /dev/null +++ b/tests/benchmarks/json_set_10M_docs_array_nested_array_text.yml @@ -0,0 +1,11 @@ +version: 0.2 +name: "json_set_10M_docs_array_nested_array_text" +description: "JSON.SET of 10M docs with a single nested array with a text field within it" + + +clientconfig: + - tool: ftsb_redisearch + - parameters: + - workers: 64 + - reporting-period: 1s + - input: "https://s3.amazonaws.com/benchmarks.redislabs/redisearch/datasets/10M-json-array-nested-array-text/10M-json-array-nested-array-text.redisjson.commands.SETUP.csv" diff --git a/tests/benchmarks/json_set_10M_docs_array_nested_dict_with_array_with_text.yml b/tests/benchmarks/json_set_10M_docs_array_nested_dict_with_array_with_text.yml new file mode 100644 index 000000000..57b646b14 --- /dev/null +++ b/tests/benchmarks/json_set_10M_docs_array_nested_dict_with_array_with_text.yml @@ -0,0 +1,11 @@ +version: 0.2 +name: "json_set_10M_docs_array_nested_dict_with_array_with_text" +description: "JSON.SET of 10M docs with a single nested array with a dict with an array with a text field within it" + + +clientconfig: + - tool: ftsb_redisearch + - parameters: + - workers: 64 + - reporting-period: 1s + - input: "https://s3.amazonaws.com/benchmarks.redislabs/redisearch/datasets/10M-json-array-nested-dict-with-array-with-text/10M-json-array-nested-dict-with-array-with-text.redisjson.commands.SETUP.csv" diff --git a/tests/benchmarks/json_set_10M_docs_array_nested_dict_with_numeric_int.yml b/tests/benchmarks/json_set_10M_docs_array_nested_dict_with_numeric_int.yml new file mode 100644 index 000000000..8c94830d4 --- /dev/null +++ b/tests/benchmarks/json_set_10M_docs_array_nested_dict_with_numeric_int.yml @@ -0,0 +1,11 @@ +version: 0.2 +name: "json_set_10M_docs_array_nested_dict_with_numeric_int" +description: "JSON.SET of 10M docs with a single nested array with a dict with an array with a numeric int field within it" + + +clientconfig: + - tool: ftsb_redisearch + - parameters: + - workers: 64 + - reporting-period: 1s + - input: "https://s3.amazonaws.com/benchmarks.redislabs/redisearch/datasets/10M-json-array-nested-dict-with-numeric-int/10M-json-array-nested-dict-with-numeric-int.redisjson.commands.SETUP.csv" diff --git a/tests/benchmarks/json_set_10M_docs_array_nested_dict_with_text.yml b/tests/benchmarks/json_set_10M_docs_array_nested_dict_with_text.yml new file mode 100644 index 000000000..3aacee567 --- /dev/null +++ b/tests/benchmarks/json_set_10M_docs_array_nested_dict_with_text.yml @@ -0,0 +1,11 @@ +version: 0.2 +name: "json_set_10M_docs_array_nested_dict_with_text" +description: "JSON.SET of 10M docs with a single nested array with a dict with text field within it" + + +clientconfig: + - tool: ftsb_redisearch + - parameters: + - workers: 64 + - reporting-period: 1s + - input: "https://s3.amazonaws.com/benchmarks.redislabs/redisearch/datasets/10M-json-array-nested-dict-with-text/10M-json-array-nested-dict-with-text.redisjson.commands.SETUP.csv" diff --git a/tests/benchmarks/json_set_10M_docs_array_single_numeric_int.yml b/tests/benchmarks/json_set_10M_docs_array_single_numeric_int.yml new file mode 100644 index 000000000..76fe8eada --- /dev/null +++ b/tests/benchmarks/json_set_10M_docs_array_single_numeric_int.yml @@ -0,0 +1,12 @@ +version: 0.2 +name: "json_set_10M_docs_array_single_numeric_int" +description: "JSON.SET of 10M docs with a single numeric int field whitin an array" + + +clientconfig: + - tool: ftsb_redisearch + - parameters: + - workers: 64 + - reporting-period: 1s + - input: "https://s3.amazonaws.com/benchmarks.redislabs/redisearch/datasets/10M-json-array-singlevalue-numeric-int/10M-json-array-singlevalue-numeric-int.redisjson.commands.SETUP.csv" +https://s3.amazonaws.com/benchmarks.redislabs/redisearch/datasets/10M-json-array-nested-array-text/10M-json-array-nested-array-text.redisjson.commands.SETUP.csv \ No newline at end of file diff --git a/tests/benchmarks/json_set_10M_docs_single_text_array.yml b/tests/benchmarks/json_set_10M_docs_array_single_text.yml similarity index 89% rename from tests/benchmarks/json_set_10M_docs_single_text_array.yml rename to tests/benchmarks/json_set_10M_docs_array_single_text.yml index 155000e94..d4347dfd2 100644 --- a/tests/benchmarks/json_set_10M_docs_single_text_array.yml +++ b/tests/benchmarks/json_set_10M_docs_array_single_text.yml @@ -1,5 +1,5 @@ version: 0.2 -name: "json_set_10M_docs_single_text_array" +name: "json_set_10M_docs_array_single_text" description: "JSON.SET of 10M docs with a single text field whitin an array" From 59e3c0e3b309da7b6775d50c4501e559cddaf002 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Tue, 30 Apr 2024 12:40:37 +0100 Subject: [PATCH 7/8] Extended the exported metrics in the benchmarks --- tests/benchmarks/defaults.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/benchmarks/defaults.yml b/tests/benchmarks/defaults.yml index b52de40f2..189063564 100644 --- a/tests/benchmarks/defaults.yml +++ b/tests/benchmarks/defaults.yml @@ -21,6 +21,10 @@ exporter: - "$.Tests.Overall.min_latency_ms" - '$."ALL STATS".*."Ops/sec"' - '$."ALL STATS".*."Latency"' + - "$.OverallRates.overallOpsRate" + - "$.OverallQuantiles.allCommands.q50" + - "$.OverallQuantiles.allCommands.q95" + - "$.OverallQuantiles.allCommands.q99" comparison: metrics: - "Ops/sec" From ee55fbc520d845679033de75e1d557c8cdeb94a4 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Tue, 30 Apr 2024 14:36:00 +0100 Subject: [PATCH 8/8] Fixed json_set_10M_docs_array_single_numeric_int benchmark definition --- tests/benchmarks/json_set_10M_docs_array_single_numeric_int.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/benchmarks/json_set_10M_docs_array_single_numeric_int.yml b/tests/benchmarks/json_set_10M_docs_array_single_numeric_int.yml index 76fe8eada..7c21d5838 100644 --- a/tests/benchmarks/json_set_10M_docs_array_single_numeric_int.yml +++ b/tests/benchmarks/json_set_10M_docs_array_single_numeric_int.yml @@ -9,4 +9,3 @@ clientconfig: - workers: 64 - reporting-period: 1s - input: "https://s3.amazonaws.com/benchmarks.redislabs/redisearch/datasets/10M-json-array-singlevalue-numeric-int/10M-json-array-singlevalue-numeric-int.redisjson.commands.SETUP.csv" -https://s3.amazonaws.com/benchmarks.redislabs/redisearch/datasets/10M-json-array-nested-array-text/10M-json-array-nested-array-text.redisjson.commands.SETUP.csv \ No newline at end of file