Skip to content

Fixes per SPEC usage of api #253

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Dec 7, 2021
Merged

Fixes per SPEC usage of api #253

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7a4f886
[fix] Fixed redis.exceptions.DataError and extended logging
filipecosta90 Dec 7, 2021
91bf9f1
[fix] Fixes per SPEC usage of api
filipecosta90 Dec 7, 2021
a24a5c9
Merge remote-tracking branch 'origin/master' into timeseries.data.error
filipecosta90 Dec 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "redisbench-admin"
version = "0.5.26"
version = "0.5.27"
description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
readme = "README.md"
Expand Down
129 changes: 60 additions & 69 deletions redisbench_admin/run/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,77 +231,68 @@ def common_exporter_logic(
testcase_metric_context_paths = []
version_target_tables = None
branch_target_tables = None
used_ts = datapoints_timestamp

if exporter_timemetric_path is not None and len(metrics) > 0:
if datapoints_timestamp is None:
# extract timestamp
datapoints_timestamp = parse_exporter_timemetric(
exporter_timemetric_path, results_dict
)
if datapoints_timestamp is None:
datapoints_timestamp = int(
datetime.datetime.now(datetime.timezone.utc).timestamp() * 1000.0
)
logging.warning(
"Error while trying to parse datapoints timestamp. Using current system timestamp Error: {}".format(
datapoints_timestamp
)
)
if (
artifact_version is not None
and artifact_version != ""
and artifact_version != "N/A"
):
# extract per-version datapoints
(
_,
per_version_time_series_dict,
version_target_tables,
) = extract_perversion_timeseries_from_results(
datapoints_timestamp,
metrics,
results_dict,
artifact_version,
tf_github_org,
tf_github_repo,
deployment_name,
deployment_type,
test_name,
tf_triggering_env,
metadata_tags,
build_variant_name,
running_platform,
testcase_metric_context_paths,
)
if tf_github_branch is not None and tf_github_branch != "":
# extract per branch datapoints
(
_,
per_branch_time_series_dict,
branch_target_tables,
) = extract_perbranch_timeseries_from_results(
datapoints_timestamp,
metrics,
results_dict,
str(tf_github_branch),
tf_github_org,
tf_github_repo,
deployment_name,
deployment_type,
test_name,
tf_triggering_env,
metadata_tags,
build_variant_name,
running_platform,
testcase_metric_context_paths,
)
else:
logging.warning(
"Requested to push data to RedisTimeSeries but no git"
" branch definition was found. git branch value {}".format(
tf_github_branch
)
if exporter_timemetric_path is not None and used_ts is None:
# extract timestamp
used_ts = parse_exporter_timemetric(exporter_timemetric_path, results_dict)

if used_ts is None:
used_ts = int(datetime.datetime.now(datetime.timezone.utc).timestamp() * 1000.0)
logging.warning(
"Error while trying to parse datapoints timestamp. Using current system timestamp Error: {}".format(
used_ts
)
)
assert used_ts != None
if (
artifact_version is not None
and artifact_version != ""
and artifact_version != "N/A"
):
# extract per-version datapoints
(
_,
per_version_time_series_dict,
version_target_tables,
) = extract_perversion_timeseries_from_results(
used_ts,
metrics,
results_dict,
artifact_version,
tf_github_org,
tf_github_repo,
deployment_name,
deployment_type,
test_name,
tf_triggering_env,
metadata_tags,
build_variant_name,
running_platform,
testcase_metric_context_paths,
)
if tf_github_branch is not None and tf_github_branch != "":
# extract per branch datapoints
(
_,
per_branch_time_series_dict,
branch_target_tables,
) = extract_perbranch_timeseries_from_results(
used_ts,
metrics,
results_dict,
str(tf_github_branch),
tf_github_org,
tf_github_repo,
deployment_name,
deployment_type,
test_name,
tf_triggering_env,
metadata_tags,
build_variant_name,
running_platform,
testcase_metric_context_paths,
)
else:
logging.error(
"Requested to push data to RedisTimeSeries but "
Expand Down
4 changes: 2 additions & 2 deletions redisbench_admin/run/redistimeseries.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,11 +303,11 @@ def timeseries_test_sucess_flow(
for (
branch_target_table_keyname,
branch_target_table_dict,
) in version_target_tables.items():
) in branch_target_tables.items():

logging.info(
"Setting target table by branch on key {}".format(
version_target_table_keyname
branch_target_table_keyname
)
)
if "contains-target" in branch_target_table_dict:
Expand Down
30 changes: 20 additions & 10 deletions redisbench_admin/utils/benchmark_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,16 +178,26 @@ def extract_redis_dbconfig_parameters(benchmark_config, dbconfig_keyname):
dbconfig_present = False
if dbconfig_keyname in benchmark_config:
dbconfig_present = True
for k in benchmark_config[dbconfig_keyname]:
if "configuration-parameters" in k:
cp = k["configuration-parameters"]
for item in cp:
for k, v in item.items():
redis_configuration_parameters[k] = v
if "dataset_load_timeout_secs" in k:
dataset_load_timeout_secs = k["dataset_load_timeout_secs"]
if "dataset_name" in k:
dataset_name = k["dataset_name"]
if type(benchmark_config[dbconfig_keyname]) == list:
for k in benchmark_config[dbconfig_keyname]:
if "configuration-parameters" in k:
cp = k["configuration-parameters"]
for item in cp:
for k, v in item.items():
redis_configuration_parameters[k] = v
if "dataset_load_timeout_secs" in k:
dataset_load_timeout_secs = k["dataset_load_timeout_secs"]
if "dataset_name" in k:
dataset_name = k["dataset_name"]
if type(benchmark_config[dbconfig_keyname]) == dict:
if "configuration-parameters" in benchmark_config[dbconfig_keyname]:
cp = benchmark_config[dbconfig_keyname]["configuration-parameters"]
for k, v in cp.items():
redis_configuration_parameters[k] = v
if "dataset_load_timeout_secs" in cp:
dataset_load_timeout_secs = cp["dataset_load_timeout_secs"]
if "dataset_name" in cp:
dataset_name = cp["dataset_name"]

return (
dbconfig_present,
Expand Down
20 changes: 14 additions & 6 deletions redisbench_admin/utils/remote.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -512,12 +512,20 @@ def push_data_to_redistimeseries(rts, time_series_dict: dict):
exporter_create_ts(rts, time_series, timeseries_name)
for timestamp, value in time_series["data"].items():
try:
rts.add(
timeseries_name,
timestamp,
value,
duplicate_policy="last",
)
if timestamp is None:
logging.warning("The provided timestamp is null. Using auto-ts")
rts.add(
timeseries_name,
value,
duplicate_policy="last",
)
else:
rts.add(
timeseries_name,
timestamp,
value,
duplicate_policy="last",
)
datapoint_inserts += 1
except redis.exceptions.DataError:
logging.warning(
Expand Down