[add] Kick off CI benchmarks automation. Included BF.ADD benchmark (#321

)

.circleci/config.yml

@@ -72,6 +72,45 @@ commands:
             mkdir -p ~/workspace/tests
             make test TEST_REPORT_DIR=$HOME/workspace/tests <<parameters.test_params>> SHOW=1
 
+  benchmark-steps:
+    parameters:
+      github_actor:
+        type: string
+        default: $CIRCLE_USERNAME
+      module_path:
+        type: string
+        default: ../../redisbloom.so
+    steps:
+      - run:
+          name: Install remote benchmark tool dependencies
+          command: |
+            VERSION=0.14.8 ./deps/readies/bin/getterraform
+      - run:
+          name: Install remote benchmark python dependencies
+          command: python3 -m pip install -r ./tests/benchmarks/requirements.txt
+      - run:
+          name: Run CI benchmarks on aws
+          timeout: 60m
+          no_output_timeout: 30m
+          command: |
+            cd ./tests/benchmarks
+            export AWS_ACCESS_KEY_ID=$PERFORMANCE_EC2_ACCESS_KEY
+            export AWS_SECRET_ACCESS_KEY=$PERFORMANCE_EC2_SECRET_KEY
+            export AWS_DEFAULT_REGION=$PERFORMANCE_EC2_REGION
+            export EC2_PRIVATE_PEM=$PERFORMANCE_EC2_PRIVATE_PEM
+
+            redisbench-admin run-remote \
+              --required-module bf \
+              --module_path << parameters.module_path >> \
+              --github_actor << parameters.github_actor >> \
+              --github_repo $CIRCLE_PROJECT_REPONAME \
+              --github_org $CIRCLE_PROJECT_USERNAME \
+              --github_sha $CIRCLE_SHA1 \
+              --github_branch $CIRCLE_BRANCH \
+              --upload_results_s3 \
+              --triggering_env circleci \
+              --push_results_redistimeseries
+
 jobs:
   lint:
     docker:
@@ -220,6 +259,17 @@ jobs:
           name: Run QA Automation
           command: MODULE_VERSION=$CIRCLE_BRANCH VERBOSE=1 TEST=nightly QUICK=1 ./tests/qa/run
 
+  performance-automation:
+    docker:
+      - image: redisfab/rmbuilder:6.2.3-x64-bionic
+    steps:
+      - checkout
+      - setup-automation
+      - run:
+          name: build artifact
+          command: make build
+      - benchmark-steps
+
 
 on-any-branch: &on-any-branch
   filters:
@@ -255,6 +305,16 @@ on-master-and-version-tags: &on-master-and-version-tags
     tags:
       only: /^v[0-9].*/
 
+on-integ-and-version-tags: &on-integ-and-version-tags
+  filters:
+    branches:
+      only:
+        - master
+        - /^\d+\.\d+.*$/
+        - /^feature-.*$/
+        - /^perf-.*$/
+    tags:
+      only: /^v[0-9].*/
 
 workflows:
   version: 2
@@ -270,6 +330,11 @@ workflows:
           <<: *on-any-branch
           requires:
             - lint
+      - performance-automation:
+          <<: *on-integ-and-version-tags
+          requires:
+            - lint
+          context: common
       - package_branch:
           <<: *on-master
           requires:
@@ -296,3 +361,5 @@ workflows:
           <<: *on-master
     jobs:
       - nightly_automation
+      - performance-automation:
+          context: common

.gitignore

@@ -21,3 +21,4 @@ deps/infer
 deps/infer*
 infer-out/
 .idea
+tests/benchmarks/profile_*

Makefile

@@ -175,3 +175,21 @@ cov coverage:
 	lcov -r $(COV_DIR)/gcov.info "*test*" "*contrib*" "*redismodule.h" "*util.c*" -o $(COV_DIR)/gcov.info > /dev/null 2>&1
 	lcov -l $(COV_DIR)/gcov.info
 	genhtml --legend -o $(COV_DIR)/report $(COV_DIR)/gcov.info > /dev/null 2>&1
+
+ifneq ($(REMOTE),)
+BENCHMARK_ARGS = run-remote
+else
+BENCHMARK_ARGS = run-local
+endif
+
+BENCHMARK_ARGS += --module_path $(realpath $(MODULE_SO)) --required-module bf
+
+ifneq ($(BENCHMARK),)
+BENCHMARK_ARGS += --test $(BENCHMARK)
+endif
+
+bench benchmark: $(MODULE_SO)
+	cd ./tests/benchmarks ;\
+	redisbench-admin $(BENCHMARK_ARGS)
+
+.PHONY: bench benchmark

tests/benchmarks/.gitignore

@@ -0,0 +1,6 @@
+*.json
+*.txt
+*.csv
+datasets/*.rdb
+*.rdb
+

tests/benchmarks/Readme.md

@@ -0,0 +1,26 @@
+# Context
+
+The automated benchmark definitions included within `tests/benchmarks` folder, provides a framework for evaluating and comparing feature branches and catching regressions prior to letting them into the master branch.
+
+To be able to run local benchmarks you need `redisbench_admin>=0.1.71` [[tool repo for full details](https://github.com/RedisLabsModules/redisbench-admin)] and the benchmark tool specified on each configuration file . You can install redisbench-admin via PyPi as any other package.
+```
+pip3 install redisbench_admin>=0.1.71
+```
+
+## Usage
+
+- Local benchmarks: `make benchmark`
+- Remote benchmarks:  `make benchmark REMOTE=1`
+
+
+## Included benchmarks
+
+Each benchmark requires a benchmark definition yaml file to present on the current directory. The benchmark spec file is fully explained on the following link: https://github.com/RedisLabsModules/redisbench-admin/tree/master/docs
+
+
+## CI integration
+
+CI benchmarks are triggered on:
+- nightly
+- pushes to branches named `master`
+- version tags

tests/benchmarks/bf_add_cap10M_err0.001.yml

@@ -0,0 +1,21 @@
+version: 0.2
+name: "bf_add_cap10M_err0.001"
+description: "Benchmarking adding an item to the Bloom Filter, 
+              with an desired false positive rate of 0.1%, and 
+              an initial capacity of 10M items"
+remote:
+ - type: oss-standalone
+ - setup: redisbloom-m5
+dbconfig:
+  - init_commands:
+    - 'BF.RESERVE largebloom 0.001 10000000'
+clientconfig:
+  - tool: redis-benchmark
+  - min-tool-version: "6.2.0"
+  - parameters:
+    - clients: 16
+    - requests: 10000000
+    - threads: 2
+    - pipeline: 1
+    - keyspacelen: 10000000
+    - command: 'bf.add largebloom __rand_int__'

tests/benchmarks/defaults.yml

@@ -0,0 +1,11 @@
+version: 0.1
+exporter:
+  redistimeseries:
+    timemetric: "$.StartTime"
+    metrics:
+      - "$.Totals.overallQuantiles.all_queries.q0"
+      - "$.Totals.overallQuantiles.all_queries.q50"
+      - "$.Totals.overallQuantiles.all_queries.q95"
+      - "$.Totals.overallQuantiles.all_queries.q99"
+      - "$.Totals.overallQuantiles.all_queries.q100"
+      - "$.Totals.overallQueryRates.all_queries"

tests/benchmarks/requirements.txt

@@ -0,0 +1 @@
+redisbench_admin>=0.2.8