Fixes for CI benchmarks: iteration 2 (#352)

* [fix] Including module artifact check on benchmarks. Included extra dev docs explaining benchmark triggering. * [fix] Fixed wrong ci benchmark name. Increased JSON.ARRAPPEND test duration now that major regression is fixed * [fix] Fixed terraform fetch script * [wip] Updated readies * [fix] fixed test naming issue on json_get_ResultSet.totalResultsAvailable_jsonsl-yahoo2_json * [wip] using readies getterraform script
RedisJSON · May 11, 2021 · fe4c64a · fe4c64a
1 parent 2422b20
commit fe4c64a
Show file tree

Hide file tree

Showing 11 changed files with 30 additions and 27 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -167,8 +167,7 @@ commands:
       - run:
           name: Install remote benchmark tool dependencies
           command: |
-            mkdir -p /workspace
-            TF_EXE_FILE_NAME=/workspace/terraform ./tests/benchmarks/remote/install_deps.sh
+            VERSION=0.14.8 ./deps/readies/bin/getterraform
       - run:
           name: Install remote benchmark python dependencies
           command: python3 -m pip install -r ./tests/benchmarks/requirements.txt
@@ -184,7 +183,7 @@ commands:
             export EC2_PRIVATE_PEM=$PERFORMANCE_EC2_PRIVATE_PEM
 
             redisbench-admin run-remote \
-              --terraform_bin_path /workspace/terraform \
+              --required-module ReJSON \
               --module_path << parameters.module_path >> \
               --github_actor << parameters.github_actor >> \
               --github_repo $CIRCLE_PROJECT_REPONAME \

diff --git a/Makefile b/Makefile
@@ -127,7 +127,8 @@ ifneq ($(REMOTE),)
 	BENCHMARK_ARGS = redisbench-admin run-remote
 endif
 
-BENCHMARK_ARGS += --module_path $(realpath $(TARGET))
+BENCHMARK_ARGS += --module_path $(realpath $(TARGET)) \
+	--required-module ReJSON
 ifneq ($(BENCHMARK),)
 	BENCHMARK_ARGS += --test $(BENCHMARK)
 endif

diff --git a/deps/readies b/deps/readies
diff --git a/tests/benchmarks/Readme.md b/tests/benchmarks/Readme.md
@@ -0,0 +1,19 @@
+# Context
+
+The automated benchmark definitions included within `tests/benchmarks` folder, provides a framework for evaluating and comparing feature branches and catching regressions prior to letting them into the master branch.
+
+To be able to run local benchmarks you need `redisbench_admin>=0.1.74` [[tool repo for full details](https://github.com/RedisLabsModules/redisbench-admin)] and the benchmark tool specified on each configuration file. You can install `redisbench-admin` via PyPi as any other package.
+```
+pip3 install redisbench_admin>=0.1.74
+```
+
+## Usage
+
+- Local benchmarks: `make benchmark`
+- Remote benchmarks:  `make benchmark REMOTE=1`
+
+
+## Included benchmarks
+
+Each benchmark requires a benchmark definition yaml file to present on the current directory. The benchmark spec file is fully explained on the following link: https://github.com/RedisLabsModules/redisbench-admin/tree/master/docs
+
diff --git a/tests/benchmarks/json_arrappend_geojson.yml b/tests/benchmarks/json_arrappend_geojson.yml
@@ -11,7 +11,7 @@ clientconfig:
   - min-tool-version: "6.2.0"
   - parameters:
     - clients: 16
-    - requests: 10000
+    - requests: 1000000
     - threads: 2
     - pipeline: 1
     - command: 'JSON.ARRAPPEND path:sonde:foo .properties.coordinateProperties "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [26.2153, 60.34085, 417.0]}, \"properties\": {\"temp\": 3.6, \"serial\": \"S0730589\", \"humidity\": 64.6, \"vel_h\": 16.0, \"uploader_callsign\": \"Lahti 24h raspberry\", \"time\": 1617220020, \"uploader\": {\"callsign\": \"Lahti 24h raspberry\"}}}"'
diff --git a/tests/benchmarks/json_get_ResultSet.totalResultsAvailable_jsonsl-yahoo2_json.yml b/tests/benchmarks/json_get_ResultSet.totalResultsAvailable_jsonsl-yahoo2_json.yml
@@ -1,5 +1,5 @@
 version: 0.2
-name: "json_set_ResultSet.totalResultsAvailable_1_jsonsl-yahoo2_json"
+name: "json_get_ResultSet.totalResultsAvailable_jsonsl-yahoo2_json"
 description: "JSON.GET jsonsl-yahoo2 ResultSet.totalResultsAvailable || https://oss.redislabs.com/redisjson/performance/"
 remote:
  - type: oss-standalone

diff --git a/tests/benchmarks/json_get_array_of_docs[1]sclr_pass_100_json.yml b/tests/benchmarks/json_get_array_of_docs[1]sclr_pass_100_json.yml
@@ -1,5 +1,5 @@
 version: 0.2
-name: "json_get_array_of_docs[1]_pass_100_json"
+name: "json_get_array_of_docs[1]sclr_pass_100_json"
 description: "JSON.GET pass-100 array_of_docs[1] || {Full document: pass-100.json https://oss.redislabs.com/redisjson/performance/"
 remote:
  - type: oss-standalone

diff --git a/tests/benchmarks/json_numincrby_num_1.yml b/tests/benchmarks/json_numincrby_num_1.yml
@@ -1,5 +1,5 @@
 version: 0.2
-name: "json_set_num_0"
+name: "json_numincrby_num_1"
 description: "JSON.NUMINCRBY num . 1 || https://oss.redislabs.com/redisjson/performance/"
 remote:
  - type: oss-standalone

diff --git a/tests/benchmarks/json_nummultby_num_2.yml b/tests/benchmarks/json_nummultby_num_2.yml
@@ -1,5 +1,5 @@
 version: 0.2
-name: "json_set_num_0"
+name: "json_nummultby_num_2"
 description: "JSON.SET num . 2 || https://oss.redislabs.com/redisjson/performance/"
 remote:
  - type: oss-standalone

diff --git a/tests/benchmarks/remote/install_deps.sh b/tests/benchmarks/remote/install_deps.sh
diff --git a/tests/benchmarks/requirements.txt b/tests/benchmarks/requirements.txt
@@ -1 +1 @@
-redisbench_admin>=0.1.63
+redisbench_admin>=0.1.74