diff --git a/README.md b/README.md index 62ceeef..11c40a7 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,9 @@ Installation is done using pip, the package installer for Python, in the followi python3 -m pip install redisbench-admin ``` +## Profiler daemon + + ## Development 1. Install [pypoetry](https://python-poetry.org/) to manage your dependencies and trigger tooling. diff --git a/docs/export.md b/docs/export.md index 980e313..e2bfc10 100644 --- a/docs/export.md +++ b/docs/export.md @@ -73,7 +73,7 @@ Optional arguments: branch/ref time-series are created (default: None) --local-dir LOCAL_DIR local dir to use as storage (default: ./) - --logname LOGNAME logname to write the logs to (default: None) + --local_file PERF_DAEMON_LOGNAME local_file to write the logs to (default: None) --github_actor [GITHUB_ACTOR] --github_repo GITHUB_REPO --github_org GITHUB_ORG diff --git a/pyproject.toml b/pyproject.toml index fb3fbb0..42ff8f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redisbench-admin" -version = "0.6.10" +version = "0.6.11" description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )." authors = ["filipecosta90 ","Redis Performance Group "] readme = "README.md" diff --git a/redisbench_admin/cli.py b/redisbench_admin/cli.py index 7af9c12..eeb4dca 100644 --- a/redisbench_admin/cli.py +++ b/redisbench_admin/cli.py @@ -71,7 +71,7 @@ def main(): "--local-dir", type=str, default="./", help="local dir to use as storage" ) parser.add_argument( - "--logname", type=str, default=None, help="logname to write the logs to" + "--logname", type=str, default=None, help="local file to write the logs to" ) if requested_tool == "run-remote": diff --git a/redisbench_admin/profilers/daemon.py b/redisbench_admin/profilers/daemon.py index 32a0e95..44dc732 100644 --- a/redisbench_admin/profilers/daemon.py +++ b/redisbench_admin/profilers/daemon.py @@ -4,27 +4,26 @@ # All rights reserved. # +import argparse +import json + # !/usr/bin/env python3 import logging - -import argparse +import os +import sys import botocore -from flask import Flask, request import daemonize -import json -import sys -import os +from flask import Flask, request from redisbench_admin.cli import populate_with_poetry_data +from redisbench_admin.profilers.perf import Perf +from redisbench_admin.profilers.perf_daemon_caller import PERF_DAEMON_LOGNAME +from redisbench_admin.profilers.profile_local import local_profilers_platform_checks from redisbench_admin.run.args import S3_BUCKET_NAME +from redisbench_admin.run.common import get_start_time_vars from redisbench_admin.run.s3 import get_test_s3_bucket_path from redisbench_admin.utils.remote import extract_git_vars - -from redisbench_admin.profilers.perf import Perf -from redisbench_admin.run.common import get_start_time_vars - -from redisbench_admin.run_local.profile_local import local_profilers_platform_checks from redisbench_admin.utils.utils import upload_artifacts_to_s3 PID_FILE = "/tmp/perfdaemon.pid" @@ -34,7 +33,6 @@ LOG_LEVEL = logging.INFO LOG_FORMAT = "%(asctime)s %(levelname)-4s %(message)s" LOG_DATEFMT = "%Y-%m-%d %H:%M:%S" -LOGNAME = "/tmp/perf-daemon.log" app = Flask(__name__) app.use_reloader = False @@ -53,8 +51,10 @@ def main(self): app.run(host="0.0.0.0", debug=False, port=5000) def set_app_loggers(self, app): - print("Writting log to {}".format(LOGNAME)) - handler = logging.handlers.RotatingFileHandler(LOGNAME, maxBytes=1024 * 1024) + print("Writting log to {}".format(PERF_DAEMON_LOGNAME)) + handler = logging.handlers.RotatingFileHandler( + PERF_DAEMON_LOGNAME, maxBytes=1024 * 1024 + ) logging.getLogger("werkzeug").setLevel(logging.DEBUG) logging.getLogger("werkzeug").addHandler(handler) app.logger.setLevel(LOG_LEVEL) diff --git a/redisbench_admin/profilers/perf.py b/redisbench_admin/profilers/perf.py index 2ad0f6a..4f4ae52 100644 --- a/redisbench_admin/profilers/perf.py +++ b/redisbench_admin/profilers/perf.py @@ -10,7 +10,6 @@ import subprocess import time - from redisbench_admin.profilers.pprof import ( PPROF_FORMAT_TEXT, run_pprof, @@ -19,7 +18,6 @@ from redisbench_admin.profilers.profilers import STACKCOLLAPSE_PATH, FLAMEGRAPH_PATH from redisbench_admin.utils.utils import whereis - PERF_CALLGRAPH_MODE_DEFAULT = "fp" LINUX_PERF_SETTINGS_MESSAGE = ( "If running in non-root user please confirm that you have:\n" diff --git a/redisbench_admin/profilers/perf_daemon_caller.py b/redisbench_admin/profilers/perf_daemon_caller.py index a259150..1c89794 100644 --- a/redisbench_admin/profilers/perf_daemon_caller.py +++ b/redisbench_admin/profilers/perf_daemon_caller.py @@ -4,10 +4,13 @@ # All rights reserved. # import logging + import requests + from redisbench_admin.utils.remote import extract_git_vars PERF_CALLGRAPH_MODE_DEFAULT = "fp" +PERF_DAEMON_LOGNAME = "/tmp/perf-daemon.log" class PerfDaemonRemoteCaller: diff --git a/redisbench_admin/profilers/pprof.py b/redisbench_admin/profilers/pprof.py index 1e2201d..386d6aa 100644 --- a/redisbench_admin/profilers/pprof.py +++ b/redisbench_admin/profilers/pprof.py @@ -6,8 +6,8 @@ import logging import os -import subprocess import re +import subprocess PPROF_FORMAT_TEXT = "-text" PPROF_FORMAT_PS = "-ps" diff --git a/redisbench_admin/run_local/profile_local.py b/redisbench_admin/profilers/profilers_local.py similarity index 80% rename from redisbench_admin/run_local/profile_local.py rename to redisbench_admin/profilers/profilers_local.py index 7f8c0ac..3f649a5 100644 --- a/redisbench_admin/run_local/profile_local.py +++ b/redisbench_admin/profilers/profilers_local.py @@ -1,110 +1,24 @@ # BSD 3-Clause License # -# Copyright (c) 2021., Redis Labs Modules +# Copyright (c) 2022., Redis Labs Modules # All rights reserved. # + +# BSD 3-Clause License +# +# import logging import platform from cpuinfo import get_cpu_info -from pytablewriter import MarkdownTableWriter from redisbench_admin.profilers.perf import Perf from redisbench_admin.profilers.vtune import Vtune from redisbench_admin.run.args import PROFILE_FREQ, MAX_PROFILERS_PER_TYPE - from redisbench_admin.run.s3 import get_test_s3_bucket_path from redisbench_admin.utils.utils import upload_artifacts_to_s3 -def local_profilers_print_artifacts_table(profilers_artifacts_matrix): - logging.info("Printing profiler generated artifacts") - test_cases = [] - profilers = [] - use_local_file = True - table_name = "Profiler artifacts" - for row in profilers_artifacts_matrix: - test_case = row[0] - profiler = row[1] - artifact = row[2] - local_file = row[3] - s3_link = row[4] - if s3_link is not None: - if len(s3_link) > 0: - use_local_file = False - - if test_case not in test_cases: - test_cases.append(test_case) - if profiler not in profilers: - profilers.append(profiler) - - # We only need to print the testcase if there are more than one - use_test_cases_row = True - - if len(test_cases) == 1: - use_test_cases_row = False - test_case = test_cases[0] - table_name = "{} for testcase {}".format(table_name, test_case) - - # We only need to print the profiler type if there are more than one - use_profilers_row = True - if len(profilers) == 1: - use_profilers_row = False - profiler = test_cases[0] - table_name = "{}. Used profiler {}".format(table_name, profiler) - - headers = [] - if use_test_cases_row: - headers.append("Test Case") - - if use_profilers_row: - headers.append("Profiler") - - headers.append("Artifact") - if use_local_file: - headers.append("Local file") - else: - headers.append("s3 link") - - profilers_final_matrix = [] - for row in profilers_artifacts_matrix: - test_case = row[0] - profiler = row[1] - artifact = row[2] - local_file = "{} ".format(row[3]) - s3_link = "{} ".format(row[4]) - - final_row = [] - if use_test_cases_row: - final_row.append(test_case) - - if use_profilers_row: - final_row.append(profiler) - - final_row.append(artifact) - if use_local_file: - final_row.append(local_file) - else: - final_row.append(s3_link) - profilers_final_matrix.append(final_row) - - writer = MarkdownTableWriter( - table_name=table_name, - headers=headers, - value_matrix=profilers_final_matrix, - ) - writer.write_table() - - -def get_profilers_rts_key_prefix(triggering_env, tf_github_org, tf_github_repo): - zset_name = "ci.benchmarks.redis.com/{triggering_env}/{github_org}/{github_repo}:profiles".format( - triggering_env=triggering_env, - github_org=tf_github_org, - github_repo=tf_github_repo, - ) - return zset_name - - def profilers_stop_if_required( args, benchmark_duration_seconds, diff --git a/redisbench_admin/profilers/profilers_schema.py b/redisbench_admin/profilers/profilers_schema.py new file mode 100644 index 0000000..2db2769 --- /dev/null +++ b/redisbench_admin/profilers/profilers_schema.py @@ -0,0 +1,96 @@ +# BSD 3-Clause License +# +# Copyright (c) 2022., Redis Labs Modules +# All rights reserved. +# +import logging + +from pytablewriter import MarkdownTableWriter + + +def get_profilers_rts_key_prefix(triggering_env, tf_github_org, tf_github_repo): + zset_name = "ci.benchmarks.redis.com/{triggering_env}/{github_org}/{github_repo}:profiles".format( + triggering_env=triggering_env, + github_org=tf_github_org, + github_repo=tf_github_repo, + ) + return zset_name + + +def local_profilers_print_artifacts_table(profilers_artifacts_matrix): + logging.info("Printing profiler generated artifacts") + test_cases = [] + profilers = [] + use_local_file = True + table_name = "Profiler artifacts" + for row in profilers_artifacts_matrix: + test_case = row[0] + profiler = row[1] + artifact = row[2] + local_file = row[3] + s3_link = row[4] + if s3_link is not None: + if len(s3_link) > 0: + use_local_file = False + + if test_case not in test_cases: + test_cases.append(test_case) + if profiler not in profilers: + profilers.append(profiler) + + # We only need to print the testcase if there are more than one + use_test_cases_row = True + + if len(test_cases) == 1: + use_test_cases_row = False + test_case = test_cases[0] + table_name = "{} for testcase {}".format(table_name, test_case) + + # We only need to print the profiler type if there are more than one + use_profilers_row = True + if len(profilers) == 1: + use_profilers_row = False + profiler = test_cases[0] + table_name = "{}. Used profiler {}".format(table_name, profiler) + + headers = [] + if use_test_cases_row: + headers.append("Test Case") + + if use_profilers_row: + headers.append("Profiler") + + headers.append("Artifact") + if use_local_file: + headers.append("Local file") + else: + headers.append("s3 link") + + profilers_final_matrix = [] + for row in profilers_artifacts_matrix: + test_case = row[0] + profiler = row[1] + artifact = row[2] + local_file = "{} ".format(row[3]) + s3_link = "{} ".format(row[4]) + + final_row = [] + if use_test_cases_row: + final_row.append(test_case) + + if use_profilers_row: + final_row.append(profiler) + + final_row.append(artifact) + if use_local_file: + final_row.append(local_file) + else: + final_row.append(s3_link) + profilers_final_matrix.append(final_row) + + writer = MarkdownTableWriter( + table_name=table_name, + headers=headers, + value_matrix=profilers_final_matrix, + ) + writer.write_table() diff --git a/redisbench_admin/run/redistimeseries.py b/redisbench_admin/run/redistimeseries.py index 6f977c1..7c9cbd4 100644 --- a/redisbench_admin/run/redistimeseries.py +++ b/redisbench_admin/run/redistimeseries.py @@ -7,12 +7,12 @@ import redis +from redisbench_admin.profilers.profilers_schema import get_profilers_rts_key_prefix from redisbench_admin.run.common import ( merge_default_and_config_metrics, common_exporter_logic, get_start_time_vars, ) -from redisbench_admin.run_local.profile_local import get_profilers_rts_key_prefix from redisbench_admin.utils.remote import ( get_project_ts_tags, get_overall_dashboard_keynames, diff --git a/redisbench_admin/run_local/run_local.py b/redisbench_admin/run_local/run_local.py index a6c7c8d..b05bab2 100644 --- a/redisbench_admin/run_local/run_local.py +++ b/redisbench_admin/run_local/run_local.py @@ -13,6 +13,9 @@ from redistimeseries.client import Client +from redisbench_admin.profilers.profilers_schema import ( + local_profilers_print_artifacts_table, +) from redisbench_admin.run.common import ( prepare_benchmark_parameters, get_start_time_vars, @@ -31,8 +34,7 @@ run_local_benchmark, check_benchmark_binaries_local_requirements, ) -from redisbench_admin.run_local.profile_local import ( - local_profilers_print_artifacts_table, +from redisbench_admin.profilers.profilers_local import ( profilers_stop_if_required, profilers_start_if_required, check_compatible_system_and_kernel_and_prepare_profile, diff --git a/redisbench_admin/run_remote/remote_failures.py b/redisbench_admin/run_remote/remote_failures.py index 1092471..73dedda 100644 --- a/redisbench_admin/run_remote/remote_failures.py +++ b/redisbench_admin/run_remote/remote_failures.py @@ -10,34 +10,34 @@ def failed_remote_run_artifact_store( - args, + upload_results_s3, client_public_ip, dirname, - full_logfile, - logname, + remote_file, + local_file, s3_bucket_name, s3_bucket_path, username, private_key, ): - local_logfile = "{}/{}".format(dirname, logname) + local_file_fullpath = "{}/{}".format(dirname, local_file) logging.error( - "The benchmark returned an error exit status. Fetching remote logfile {} into {}".format( - full_logfile, local_logfile + "The benchmark returned an error exit status. Fetching remote file {} into {}".format( + remote_file, local_file_fullpath ) ) fetch_file_from_remote_setup( client_public_ip, username, private_key, - local_logfile, - full_logfile, + local_file_fullpath, + remote_file, ) - if args.upload_results_s3: + if upload_results_s3: logging.info( - "Uploading logfile {} to s3. s3 bucket name: {}. s3 bucket path: {}".format( - local_logfile, s3_bucket_name, s3_bucket_path + "Uploading file {} to s3. s3 bucket name: {}. s3 bucket path: {}".format( + local_file_fullpath, s3_bucket_name, s3_bucket_path ) ) - artifacts = [local_logfile] + artifacts = [local_file_fullpath] upload_artifacts_to_s3(artifacts, s3_bucket_name, s3_bucket_path) diff --git a/redisbench_admin/run_remote/run_remote.py b/redisbench_admin/run_remote/run_remote.py index 15693d3..cdd17ce 100644 --- a/redisbench_admin/run_remote/run_remote.py +++ b/redisbench_admin/run_remote/run_remote.py @@ -11,7 +11,10 @@ from pytablewriter import MarkdownTableWriter from redistimeseries.client import Client -from redisbench_admin.profilers.perf_daemon_caller import PerfDaemonRemoteCaller +from redisbench_admin.profilers.perf_daemon_caller import ( + PerfDaemonRemoteCaller, + PERF_DAEMON_LOGNAME, +) from redisbench_admin.run.args import PROFILE_FREQ from redisbench_admin.run.common import ( get_start_time_vars, @@ -470,7 +473,24 @@ def run_remote_command_logic(args, project_name, project_version): logging.info("Stopping remote profiler") profiler_result = remote_perf.stop_profile() if profiler_result is False: - logging.error("Unsuccessful profiler stop") + logging.error( + "Unsuccessful profiler stop." + + " Fetching remote perf-daemon logfile {}".format( + PERF_DAEMON_LOGNAME + ) + ) + failed_remote_run_artifact_store( + args.upload_results_s3, + server_public_ip, + dirname, + PERF_DAEMON_LOGNAME, + logname, + s3_bucket_name, + s3_bucket_path, + username, + private_key, + ) + return_code |= 1 ( perf_stop_status, profile_artifacts, @@ -557,8 +577,8 @@ def run_remote_command_logic(args, project_name, project_version): if remote_run_result is False: failed_remote_run_artifact_store( - args, - client_public_ip, + args.upload_results_s3, + server_public_ip, dirname, full_logfiles[0], logname, diff --git a/redisbench_admin/utils/remote.py b/redisbench_admin/utils/remote.py index 9d8a957..3201c50 100644 --- a/redisbench_admin/utils/remote.py +++ b/redisbench_admin/utils/remote.py @@ -553,7 +553,7 @@ def exporter_create_ts(rts, time_series, timeseries_name): timeseries_name, time_series["labels"] ) ) - rts.create(timeseries_name, labels=time_series["labels"]) + rts.create(timeseries_name, labels=time_series["labels"], chunk_size=128) except redis.exceptions.ResponseError as e: if "already exists" in e.__str__(): logging.debug( diff --git a/tests/test_run_local.py b/tests/test_run_local.py index 4969a01..4e2a49b 100644 --- a/tests/test_run_local.py +++ b/tests/test_run_local.py @@ -10,7 +10,7 @@ from redisbench_admin.run_local.local_helpers import ( check_benchmark_binaries_local_requirements, ) -from redisbench_admin.run_local.profile_local import get_profilers_rts_key_prefix +from redisbench_admin.profilers.profilers_schema import get_profilers_rts_key_prefix from redisbench_admin.run_local.run_local import ( run_local_command_logic, )