From 4a909ded3b25341f13267c62f3e368750f3c406f Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Thu, 24 Feb 2022 15:02:45 +0000 Subject: [PATCH 1/3] Added option for in case of failure exit immediately. --- redisbench_admin/run/args.py | 9 +++++++++ redisbench_admin/run_local/run_local.py | 1 + redisbench_admin/run_remote/run_remote.py | 7 +++++++ redisbench_admin/utils/benchmark_config.py | 9 ++++++--- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/redisbench_admin/run/args.py b/redisbench_admin/run/args.py index 2771a18..9a2e200 100644 --- a/redisbench_admin/run/args.py +++ b/redisbench_admin/run/args.py @@ -25,6 +25,7 @@ SETUP = os.getenv("SETUP", "") S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME", "ci.benchmarks.redislabs") PUSH_S3 = bool(os.getenv("PUSH_S3", False)) +FAIL_FAST = bool(os.getenv("FAIL_FAST", False)) PROFILERS_DSO = os.getenv("PROFILERS_DSO", None) PROFILERS_ENABLED = bool(int(os.getenv("PROFILE", 0))) PROFILERS = os.getenv("PROFILERS", PROFILERS_DEFAULT) @@ -41,6 +42,14 @@ def common_run_args(parser): action="store_true", help="Keep environment and topology up after benchmark.", ) + parser.add_argument( + "--fail_fast", + required=False, + default=FAIL_FAST, + action="store_true", + help="In case of failure exit immediately. Otherwise run all other tests and then exit on error.", + ) + parser.add_argument( "--dbdir_folder", type=str, diff --git a/redisbench_admin/run_local/run_local.py b/redisbench_admin/run_local/run_local.py index b05bab2..af23bf9 100644 --- a/redisbench_admin/run_local/run_local.py +++ b/redisbench_admin/run_local/run_local.py @@ -114,6 +114,7 @@ def run_local_command_logic(args, project_name, project_version): ) ( + _, benchmark_definitions, default_metrics, _, diff --git a/redisbench_admin/run_remote/run_remote.py b/redisbench_admin/run_remote/run_remote.py index 992372d..335df02 100644 --- a/redisbench_admin/run_remote/run_remote.py +++ b/redisbench_admin/run_remote/run_remote.py @@ -122,6 +122,7 @@ def run_remote_command_logic(args, project_name, project_version): ssh_pem_check(EC2_PRIVATE_PEM, private_key) ( + benchmark_defs_result, benchmark_definitions, default_metrics, exporter_timemetric_path, @@ -130,6 +131,12 @@ def run_remote_command_logic(args, project_name, project_version): ) = prepare_benchmark_definitions(args) return_code = 0 + if benchmark_defs_result is False and args.fail_fast: + logging.critical( + "Detected errors while preparing benchmark definitions. Exiting right away!" + ) + exit(1) + remote_envs = {} dirname = "." ( diff --git a/redisbench_admin/utils/benchmark_config.py b/redisbench_admin/utils/benchmark_config.py index 904ae1f..15213fe 100644 --- a/redisbench_admin/utils/benchmark_config.py +++ b/redisbench_admin/utils/benchmark_config.py @@ -51,6 +51,7 @@ def parse_exporter_timemetric(metric_path: str, results_dict: dict): def prepare_benchmark_definitions(args): benchmark_definitions = {} + result = True defaults_filename, files = get_testfiles_to_process(args) ( @@ -61,13 +62,15 @@ def prepare_benchmark_definitions(args): clusterconfig, ) = get_defaults(defaults_filename) for usecase_filename in files: - with open(usecase_filename, "r") as stream: - result, benchmark_config, test_name = get_final_benchmark_config( + with open(usecase_filename, "r", encoding="utf8") as stream: + test_result, benchmark_config, test_name = get_final_benchmark_config( default_kpis, stream, usecase_filename ) - if result: + result &= test_result + if test_result: benchmark_definitions[test_name] = benchmark_config return ( + result, benchmark_definitions, default_metrics, exporter_timemetric_path, From 90046036cd587ec6fac44e922b9cad2286bba37c Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Thu, 24 Feb 2022 15:03:38 +0000 Subject: [PATCH 2/3] in the case of a bad benchmark config exit with error --- redisbench_admin/run_remote/run_remote.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/redisbench_admin/run_remote/run_remote.py b/redisbench_admin/run_remote/run_remote.py index 335df02..6c5ea55 100644 --- a/redisbench_admin/run_remote/run_remote.py +++ b/redisbench_admin/run_remote/run_remote.py @@ -131,11 +131,13 @@ def run_remote_command_logic(args, project_name, project_version): ) = prepare_benchmark_definitions(args) return_code = 0 - if benchmark_defs_result is False and args.fail_fast: - logging.critical( - "Detected errors while preparing benchmark definitions. Exiting right away!" - ) - exit(1) + if benchmark_defs_result is False: + return_code = 1 + if args.fail_fast: + logging.critical( + "Detected errors while preparing benchmark definitions. Exiting right away!" + ) + exit(1) remote_envs = {} dirname = "." From 78cf7402f8ca1c1aa90574318c11c10b4fc79a4f Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Thu, 24 Feb 2022 15:04:07 +0000 Subject: [PATCH 3/3] Bumping version from 0.6.23 to 0.6.24 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 83fe9e9..701648e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redisbench-admin" -version = "0.6.23" +version = "0.6.24" description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )." authors = ["filipecosta90 ","Redis Performance Group "] readme = "README.md"