ann-benchmarks support

.flake8

@@ -10,4 +10,5 @@ exclude =
     *.egg-info,
     .nox,
     .pytest_cache,
-    .mypy_cache
+    .mypy_cache,
+    redisbench_admin/run/ann/pkg/

.github/workflows/publish-pypi.yml

@@ -33,6 +33,10 @@ jobs:
           poetry config virtualenvs.in-project false
           poetry config virtualenvs.path ~/.virtualenvs
 
+      - name: Set Submodules
+        run: |
+          git submodule update --init --recursive
+
       - name: Install Dependencies
         run: poetry install
         if: steps.cache.outputs.cache-hit != 'true'

.github/workflows/tox.yml

@@ -34,6 +34,10 @@ jobs:
       run: |
         pip3 install black coverage flake8 tox tox-docker tox-poetry
 
+    - name: Set Submodules
+      run: |
+        git submodule update --init --recursive
+
     - name: Install Redis
       run: |
         sudo add-apt-repository ppa:redislabs/redis

.gitignore

@@ -1,5 +1,8 @@
 # redisbench-admin related inputs/outputs
 
+redisbench_admin/run/ann/pkg/
+
+
 *.json
 *.tar.gz
 *.csv

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "redisbench_admin/run/ann/pkg"]
+	path = redisbench_admin/run/ann/pkg
+	url = https://github.com/RedisAI/ann-benchmarks

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.6.26"
+version = "0.6.27"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "README.md"
@@ -14,12 +14,10 @@ python = "^3.6.1"
 humanize = "^2.4.0"
 requests = "^2.23.0"
 py_cpuinfo = "^5.0.0"
-redis = "^3.5.3"
+redis = "^4.1.4"
 boto3 = "^1.13.24"
 tqdm = "^4.46.1"
 toml = "^0.10.1"
-redistimeseries = "^1.4.3"
-redis-py-cluster = "^2.1.0"
 paramiko = "^2.7.2"
 jsonpath_ng = "^1.5.2"
 pysftp = "^0.2.9"

redisbench_admin/compare/compare.py

@@ -11,7 +11,7 @@
 import datetime as dt
 
 from redisbench_admin.utils.remote import get_overall_dashboard_keynames
-from redistimeseries.client import Client
+
 
 from redisbench_admin.utils.utils import get_ts_metric_name
 
@@ -29,13 +29,13 @@ def compare_command_logic(args, project_name, project_version):
             args.redistimeseries_port,
         )
     )
-    rts = Client(
+    rts = redis.Redis(
         host=args.redistimeseries_host,
         port=args.redistimeseries_port,
         password=args.redistimeseries_pass,
         username=args.redistimeseries_user,
     )
-    rts.redis.ping()
+    rts.ping()
 
     tf_github_org = args.github_org
     tf_github_repo = args.github_repo
@@ -108,7 +108,7 @@ def compare_command_logic(args, project_name, project_version):
     ) = get_overall_dashboard_keynames(tf_github_org, tf_github_repo, tf_triggering_env)
     test_names = []
     try:
-        test_names = rts.redis.smembers(testcases_setname)
+        test_names = rts.smembers(testcases_setname)
         test_names = list(test_names)
         test_names.sort()
     except redis.exceptions.ResponseError as e:

redisbench_admin/environments/oss_cluster.py

@@ -44,7 +44,7 @@ def spin_up_local_redis_cluster(
             )
         )
         redis_process = subprocess.Popen(command)
-        r = redis.StrictRedis(port=shard_port)
+        r = redis.Redis(port=shard_port)
         result = wait_for_conn(r, dataset_load_timeout_secs)
         if result is True:
             logging.info("Redis available. pid={}".format(redis_process.pid))
@@ -164,7 +164,7 @@ def generate_cluster_redis_server_args(
         "--cluster-config-file",
         "cluster-node-port-{}.config".format(port),
         "--save",
-        '""',
+        "''",
         "--cluster-announce-ip",
         "{}".format(ip),
         "--port",

redisbench_admin/environments/oss_standalone.py

@@ -36,7 +36,7 @@ def spin_up_local_redis(
         )
     )
     redis_process = subprocess.Popen(command)
-    result = wait_for_conn(redis.StrictRedis(port=port), dataset_load_timeout_secs)
+    result = wait_for_conn(redis.Redis(port=port), dataset_load_timeout_secs)
     if result is True:
         logging.info("Redis available")
     return [redis_process]
@@ -54,7 +54,7 @@ def generate_standalone_redis_server_args(
     command = [
         binary,
         "--save",
-        '""',
+        "",
         "--port",
         "{}".format(port),
         "--dir",

redisbench_admin/export/export.py

@@ -9,7 +9,7 @@
 import datetime
 
 import redis
-from redistimeseries.client import Client
+
 
 from redisbench_admin.export.common.common import split_tags_string
 from redisbench_admin.run.git import git_vars_crosscheck
@@ -118,13 +118,13 @@ def export_command_logic(args, project_name, project_version):
             args.redistimeseries_host, args.redistimeseries_port
         )
     )
-    rts = Client(
+    rts = redis.Redis(
         host=args.redistimeseries_host,
         port=args.redistimeseries_port,
         password=args.redistimeseries_pass,
     )
     try:
-        rts.redis.ping()
+        rts.ping()
     except redis.exceptions.ConnectionError as e:
         logging.error(
             "Error while connecting to RedisTimeSeries data sink at: {}:{}. Error: {}".format(

redisbench_admin/grafana_api/app.py

@@ -15,7 +15,7 @@ def create_app(conn, auth_server_host, auth_server_port, test_config=None):
     def verify_password(username, password):
         result = False
         try:
-            auth_server_conn = redis.StrictRedis(
+            auth_server_conn = redis.Redis(
                 host=auth_server_host,
                 port=auth_server_port,
                 decode_responses=True,

redisbench_admin/grafana_api/grafana_api.py

@@ -27,7 +27,7 @@ def grafana_api_command_logic(args, project_name, project_version):
         )
     )
 
-    conn = redis.StrictRedis(
+    conn = redis.Redis(
         host=args.redis_host,
         port=args.redis_port,
         decode_responses=True,

redisbench_admin/profilers/perf.py

@@ -301,6 +301,12 @@ def generate_trace_file_from_profile(self, filename=None):
                                     self.perf, e.__str__()
                                 )
                             )
+                        except Exception as e:
+                            self.logger.error(
+                                "Unable to run {} script {}".format(
+                                    self.perf, e.__str__()
+                                )
+                            )
                 else:
                     logging.warning(
                         "Unable to generate main thread only collapsed stack files given no PID data is available"

redisbench_admin/run/ann/__init__.py

@@ -0,0 +1,5 @@
+#  Apache License Version 2.0
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#

redisbench_admin/run/ann/ann.py

@@ -0,0 +1,35 @@
+import pkg_resources
+
+ANN_MULTIRUN_PATH = pkg_resources.resource_filename(
+    "redisbench_admin", "run/ann/pkg/multirun.py"
+)
+
+
+def prepare_ann_benchmark_command(
+    server_private_ip: object,
+    server_plaintext_port: object,
+    cluster_mode: bool,
+    benchmark_config: object,
+    results_file: str,
+    current_workdir: str,
+):
+    command_arr = ["python3", ANN_MULTIRUN_PATH]
+
+    if "arguments" in benchmark_config:
+        command_arr.extend(benchmark_config["arguments"].strip().split(" "))
+
+    if "parameters" in benchmark_config:
+        for k, v in benchmark_config["parameters"].items():
+            command_arr.extend(["--{}".format(k), str(v)])
+
+    if server_private_ip is not None:
+        command_arr.extend(["--host", "{}".format(server_private_ip)])
+    if server_plaintext_port is not None:
+        command_arr.extend(["--port", str(server_plaintext_port)])
+    if cluster_mode:
+        command_arr.append("--cluster")
+
+    command_arr.extend(["--json-output", "{}/{}".format(current_workdir, results_file)])
+
+    command_str = " ".join([str(x) for x in command_arr])
+    return command_arr, command_str

redisbench_admin/run/args.py

@@ -62,7 +62,7 @@ def common_run_args(parser):
         default="redis-benchmark,redisgraph-benchmark-go,ycsb,"
         + "tsbs_run_queries_redistimeseries,tsbs_load_redistimeseries,"
         + "ftsb_redisearch,"
-        + "aibench_run_inference_redisai_vision",
+        + "aibench_run_inference_redisai_vision,ann-benchmarks",
         help="comma separated list of allowed tools for this module. By default all the supported are allowed.",
     )
     parser.add_argument(

redisbench_admin/run/common.py

@@ -15,6 +15,7 @@
 from redisbench_admin.run.aibench_run_inference_redisai_vision.aibench_run_inference_redisai_vision import (
     prepare_aibench_benchmark_command,
 )
+from redisbench_admin.run.ann.ann import prepare_ann_benchmark_command
 from redisbench_admin.run.ftsb.ftsb import prepare_ftsb_benchmark_command
 from redisbench_admin.run.memtier_benchmark.memtier_benchmark import (
     prepare_memtier_benchmark_command,
@@ -176,6 +177,15 @@ def prepare_benchmark_parameters_specif_tooling(
             cluster_api_enabled,
             remote_results_file,
         )
+    if "ann" in benchmark_tool:
+        (command_arr, command_str,) = prepare_ann_benchmark_command(
+            server_private_ip,
+            server_plaintext_port,
+            cluster_api_enabled,
+            entry,
+            remote_results_file,
+            current_workdir,
+        )
     if "ftsb_" in benchmark_tool:
         input_data_file = None
         if isremote is True:

redisbench_admin/run/grafana.py

@@ -106,27 +106,27 @@ def generate_artifacts_table_grafana_redis(
     if args.push_results_redistimeseries:
         current_time = time.time() * 1000
         timeframe_by_branch = current_time - EXPIRE_TIME_MSECS_PROFILE_KEYS
-        rts.redis.zadd(
+        rts.zadd(
             zset_profiles_setups_testcases_branches,
             {tf_github_branch: start_time_ms},
         )
-        rts.redis.zadd(
+        rts.zadd(
             zset_profiles_setups_testcases_branches_latest_link,
             {https_link: start_time_ms},
         )
-        rts.redis.zadd(
+        rts.zadd(
             zset_profiles_setup,
             {setup_name: start_time_ms},
         )
-        rts.redis.zadd(
+        rts.zadd(
             zset_profiles_setups_testcases,
             {test_name: start_time_ms},
         )
-        rts.redis.zadd(
+        rts.zadd(
             zset_profiles_setups_testcases_profileid,
             {profile_id: start_time_ms},
         )
-        rts.redis.zadd(
+        rts.zadd(
             zset_profiles,
             {profile_id: start_time_ms},
         )
@@ -138,11 +138,11 @@ def generate_artifacts_table_grafana_redis(
             zset_profiles_setups_testcases_branches_latest_link,
         ]
         for keyname in sorted_set_keys:
-            rts.redis.zremrangebyscore(keyname, 0, int(timeframe_by_branch))
+            rts.zremrangebyscore(keyname, 0, int(timeframe_by_branch))
 
-        rts.redis.sadd(profile_set_redis_key, test_name)
-        rts.redis.expire(profile_set_redis_key, EXPIRE_TIME_SECS_PROFILE_KEYS)
-        rts.redis.setex(
+        rts.sadd(profile_set_redis_key, test_name)
+        rts.expire(profile_set_redis_key, EXPIRE_TIME_SECS_PROFILE_KEYS)
+        rts.setex(
             profile_string_testcase_markdown_key,
             EXPIRE_TIME_SECS_PROFILE_KEYS,
             profile_markdown_str,