diff --git a/pyproject.toml b/pyproject.toml index df19a61..92adb69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redisbench-admin" -version = "0.2.6" +version = "0.2.7" description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )." authors = ["filipecosta90 "] readme = "README.md" diff --git a/redisbench_admin/cli.py b/redisbench_admin/cli.py index 3da37c4..552e37b 100644 --- a/redisbench_admin/cli.py +++ b/redisbench_admin/cli.py @@ -6,6 +6,7 @@ import argparse import logging +import os import sys import toml @@ -17,12 +18,19 @@ from redisbench_admin.extract.extract import extract_command_logic from redisbench_admin.run_local.args import create_run_local_arguments from redisbench_admin.run_local.run_local import run_local_command_logic -from redisbench_admin.run_remote.args import create_run_remote_arguments, LOG_LEVEL +from redisbench_admin.run_remote.args import create_run_remote_arguments from redisbench_admin.run_remote.run_remote import run_remote_command_logic from redisbench_admin.watchdog.args import create_watchdog_arguments from redisbench_admin.watchdog.watchdog import watchdog_command_logic +LOG_LEVEL = logging.INFO +if os.getenv("VERBOSE", "1") == "0": + LOG_LEVEL = logging.WARN +LOG_FORMAT = "%(asctime)s %(levelname)-4s %(message)s" +LOG_DATEFMT = "%Y-%m-%d %H:%M:%S" + + def populate_with_poetry_data(): project_name = "redisbench-admin" project_version = __version__ @@ -38,14 +46,6 @@ def populate_with_poetry_data(): return project_name, project_description, project_version -# logging settings -logging.basicConfig( - format="%(asctime)s %(levelname)-4s %(message)s", - level=LOG_LEVEL, - datefmt="%Y-%m-%d %H:%M:%S", -) - - def main(): if len(sys.argv) < 2: print( @@ -66,6 +66,9 @@ def main(): parser.add_argument( "--local-dir", type=str, default="./", help="local dir to use as storage" ) + parser.add_argument( + "--logname", type=str, default=None, help="logname to write the logs to" + ) if requested_tool == "run-remote": parser = create_run_remote_arguments(parser) @@ -96,6 +99,24 @@ def main(): argv = sys.argv[2:] args = parser.parse_args(args=argv) + + if args.logname is not None: + print("Writting log to {}".format(args.logname)) + logging.basicConfig( + filename=args.logname, + filemode="a", + format=LOG_FORMAT, + datefmt=LOG_DATEFMT, + level=LOG_LEVEL, + ) + else: + # logging settings + logging.basicConfig( + format=LOG_FORMAT, + level=LOG_LEVEL, + datefmt=LOG_DATEFMT, + ) + if requested_tool == "run-local": run_local_command_logic(args) if requested_tool == "run-remote": diff --git a/redisbench_admin/run_local/run_local.py b/redisbench_admin/run_local/run_local.py index c7d92cc..9b63037 100644 --- a/redisbench_admin/run_local/run_local.py +++ b/redisbench_admin/run_local/run_local.py @@ -31,7 +31,7 @@ extract_benchmark_tool_settings, check_required_modules, results_dict_kpi_check, - extract_redis_configuration_parameters, + extract_redis_dbconfig_parameters, ) from redisbench_admin.run.redis_benchmark.redis_benchmark import ( redis_benchmark_ensure_min_version_local, @@ -134,7 +134,7 @@ def run_local_command_logic(args): ) check_dataset_local_requirements(benchmark_config, temporary_dir, dirname) - redis_configuration_parameters = extract_redis_configuration_parameters( + redis_configuration_parameters, _ = extract_redis_dbconfig_parameters( benchmark_config, "dbconfig" ) @@ -158,7 +158,17 @@ def run_local_command_logic(args): check_required_modules(module_names, required_modules) # run initialization commands before benchmark starts + logging.info("Running initialization commands before benchmark starts.") + execute_init_commands_start_time = datetime.datetime.now() execute_init_commands(benchmark_config, r) + execute_init_commands_duration_seconds = ( + datetime.datetime.now() - execute_init_commands_start_time + ).seconds + logging.info( + "Running initialization commands took {} secs.".format( + execute_init_commands_duration_seconds + ) + ) # setup the benchmark start_time, start_time_ms, start_time_str = get_start_time_vars() diff --git a/redisbench_admin/run_remote/args.py b/redisbench_admin/run_remote/args.py index 4541f39..cf197da 100644 --- a/redisbench_admin/run_remote/args.py +++ b/redisbench_admin/run_remote/args.py @@ -3,7 +3,6 @@ # Copyright (c) 2021., Redis Labs Modules # All rights reserved. # -import logging import os import socket @@ -16,9 +15,6 @@ PERFORMANCE_RTS_PUSH, ) -LOG_LEVEL = logging.INFO -if os.getenv("VERBOSE", "1") == "0": - LOG_LEVEL = logging.WARN DEFAULT_TRIGGERING_ENV = socket.gethostname() TRIGGERING_ENV = os.getenv("TRIGGERING_ENV", DEFAULT_TRIGGERING_ENV) diff --git a/redisbench_admin/run_remote/run_remote.py b/redisbench_admin/run_remote/run_remote.py index 7b50fdd..7daf85d 100644 --- a/redisbench_admin/run_remote/run_remote.py +++ b/redisbench_admin/run_remote/run_remote.py @@ -34,7 +34,7 @@ prepare_benchmark_definitions, check_required_modules, results_dict_kpi_check, - extract_redis_configuration_parameters, + extract_redis_dbconfig_parameters, ) from redisbench_admin.utils.redisgraph_benchmark_go import ( spin_up_standalone_remote_redis, @@ -359,11 +359,10 @@ def run_remote_command_logic(args): # in case of some unexpected error we fail the test try: - redis_configuration_parameters = ( - extract_redis_configuration_parameters( - benchmark_config, "dbconfig" - ) - ) + ( + redis_configuration_parameters, + dataset_load_timeout_secs, + ) = extract_redis_dbconfig_parameters(benchmark_config, "dbconfig") # setup Redis spin_up_standalone_remote_redis( benchmark_config, @@ -383,7 +382,7 @@ def run_remote_command_logic(args): server_public_ip, username, ) - result = wait_for_conn(local_redis_conn) + result = wait_for_conn(local_redis_conn, dataset_load_timeout_secs) dataset_load_end_time = datetime.datetime.now() if result is True: logging.info("Redis available") @@ -409,7 +408,19 @@ def run_remote_command_logic(args): check_required_modules(module_names, required_modules) # run initialization commands before benchmark starts + logging.info( + "Running initialization commands before benchmark starts." + ) + execute_init_commands_start_time = datetime.datetime.now() execute_init_commands(benchmark_config, local_redis_conn) + execute_init_commands_duration_seconds = ( + datetime.datetime.now() - execute_init_commands_start_time + ).seconds + logging.info( + "Running initialization commands took {} secs.".format( + execute_init_commands_duration_seconds + ) + ) ssh_tunnel.close() # Close the tunnel diff --git a/redisbench_admin/utils/benchmark_config.py b/redisbench_admin/utils/benchmark_config.py index 91c1369..e6a41e3 100644 --- a/redisbench_admin/utils/benchmark_config.py +++ b/redisbench_admin/utils/benchmark_config.py @@ -123,8 +123,9 @@ def merge_default_and_specific_properties_dict_type( ) -def extract_redis_configuration_parameters(benchmark_config, dbconfig_keyname): +def extract_redis_dbconfig_parameters(benchmark_config, dbconfig_keyname): redis_configuration_parameters = {} + dataset_load_timeout_secs = 120 if dbconfig_keyname in benchmark_config: for k in benchmark_config[dbconfig_keyname]: if "configuration-parameters" in k: @@ -132,8 +133,10 @@ def extract_redis_configuration_parameters(benchmark_config, dbconfig_keyname): for item in cp: for k, v in item.items(): redis_configuration_parameters[k] = v + if "dataset_load_timeout_secs" in k: + dataset_load_timeout_secs = k["dataset_load_timeout_secs"] - return redis_configuration_parameters + return redis_configuration_parameters, dataset_load_timeout_secs def process_default_yaml_properties_file( diff --git a/redisbench_admin/utils/utils.py b/redisbench_admin/utils/utils.py index 217b458..e2b265f 100644 --- a/redisbench_admin/utils/utils.py +++ b/redisbench_admin/utils/utils.py @@ -244,4 +244,8 @@ def wait_for_conn(conn, retries=120, command="PING", should_be=True): time.sleep(1) retries -= 1 logging.debug("Waiting for Redis") + if retries == 0: + logging.debug( + "Redis busy loading time surpassed the timeout of {} secs".format(retries) + ) return result diff --git a/redisbench_admin/watchdog/watchdog.py b/redisbench_admin/watchdog/watchdog.py index 620cdb5..ef2127c 100644 --- a/redisbench_admin/watchdog/watchdog.py +++ b/redisbench_admin/watchdog/watchdog.py @@ -10,6 +10,7 @@ import time import boto3 +import redis from redistimeseries.client import Client from redisbench_admin.run.common import get_start_time_vars @@ -125,14 +126,17 @@ def watchdog_command_logic(args): running_count, _ = get_ci_ec2_instances_by_state( ec2_client, ci_machines_prefix, "running" ) - - rts.add( - tsname_overall_running, - start_time_ms, - running_count, - labels={"cloud": cloud, "region": EC2_REGION}, - ) - + try: + rts.add( + tsname_overall_running, + start_time_ms, + running_count, + labels={"cloud": cloud, "region": EC2_REGION}, + ) + except redis.exceptions.ConnectionError as e: + logging.error( + "Detected an error while writing data to rts: {}".format(e.__str__()) + ) sleep_time_secs = float(update_interval) - ( (datetime.datetime.now() - starttime).total_seconds() % float(update_interval) diff --git a/tests/test_benchmark_config.py b/tests/test_benchmark_config.py index 692ebab..527a605 100644 --- a/tests/test_benchmark_config.py +++ b/tests/test_benchmark_config.py @@ -5,7 +5,7 @@ from redisbench_admin.utils.benchmark_config import ( results_dict_kpi_check, check_required_modules, - extract_redis_configuration_parameters, + extract_redis_dbconfig_parameters, ) @@ -51,19 +51,44 @@ def test_extract_redis_configuration_parameters(): "./tests/test_data/redisgraph-benchmark-go-defaults.yml", "r" ) as config_fd: benchmark_config = yaml.safe_load(config_fd) - redis_configuration_parameters = extract_redis_configuration_parameters( - benchmark_config, "dbconfig" - ) + ( + redis_configuration_parameters, + dataset_load_timeout_secs, + ) = extract_redis_dbconfig_parameters(benchmark_config, "dbconfig") assert redis_configuration_parameters == {} + assert dataset_load_timeout_secs == 120 with open( "./tests/test_data/tsbs-devops-ingestion-scale100-4days-keyspace.yml", "r" ) as config_fd: benchmark_config = yaml.safe_load(config_fd) - redis_configuration_parameters = extract_redis_configuration_parameters( - benchmark_config, "dbconfig" - ) + ( + redis_configuration_parameters, + dataset_load_timeout_secs, + ) = extract_redis_dbconfig_parameters(benchmark_config, "dbconfig") + assert dataset_load_timeout_secs == 120 assert redis_configuration_parameters == { "notify-keyspace-events": "KEA", "timeout": 0, } + + with open( + "./tests/test_data/redisgraph-benchmark-go-defaults.yml", "r" + ) as config_fd: + benchmark_config = yaml.safe_load(config_fd) + ( + redis_configuration_parameters, + dataset_load_timeout_secs, + ) = extract_redis_dbconfig_parameters(benchmark_config, "dbconfig") + assert redis_configuration_parameters == {} + assert dataset_load_timeout_secs == 120 + + with open( + "./tests/test_data/redisgraph-benchmark-go-dataset-timeout.yml", "r" + ) as config_fd: + benchmark_config = yaml.safe_load(config_fd) + ( + redis_configuration_parameters, + dataset_load_timeout_secs, + ) = extract_redis_dbconfig_parameters(benchmark_config, "dbconfig") + assert dataset_load_timeout_secs == 1200 diff --git a/tests/test_data/redisgraph-benchmark-go-dataset-timeout.yml b/tests/test_data/redisgraph-benchmark-go-dataset-timeout.yml new file mode 100644 index 0000000..f2d7f60 --- /dev/null +++ b/tests/test_data/redisgraph-benchmark-go-dataset-timeout.yml @@ -0,0 +1,21 @@ +name: "UPDATE-BASELINE" +remote: + - setup: redisgraph-r5 + - type: oss-standalone +dbconfig: + - dataset: "datasets/single_node.rdb" + - dataset_load_timeout_secs: 1200 +clientconfig: + - tool: redisgraph-benchmark-go + - parameters: + - graph: "g" + - rps: 0 + - clients: 32 + - threads: 4 + - connections: 32 + - requests: 1000000 + - queries: + - { q: "MATCH (n) WHERE ID(n) = 0 SET n.v = n.v + 1", ratio: 1 } +kpis: + - le: { $.OverallClientLatencies.Total.q50: 2.0 } + - ge: { $.OverallQueryRates.Total: 18000 }