From 515dc092a803505448ddc1ed0b692df2ad854daf Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Tue, 24 Oct 2023 00:01:28 +0000 Subject: [PATCH 01/16] add loss metric to min_eval_metrics registry --- scoring/scoring.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scoring/scoring.py b/scoring/scoring.py index 12aae1357..3fd2a5f83 100644 --- a/scoring/scoring.py +++ b/scoring/scoring.py @@ -47,6 +47,7 @@ 'ctc_loss', 'wer', 'l1_loss', + 'loss', ] MAX_EVAL_METRICS = ['average_precision', 'ssim', 'accuracy', 'bleu_score'] From 1b8c1dc7d529d596a938ee1829394a9d11840f25 Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Tue, 24 Oct 2023 00:12:10 +0000 Subject: [PATCH 02/16] debugging --- scoring/scoring.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scoring/scoring.py b/scoring/scoring.py index 3fd2a5f83..959965d8d 100644 --- a/scoring/scoring.py +++ b/scoring/scoring.py @@ -47,7 +47,6 @@ 'ctc_loss', 'wer', 'l1_loss', - 'loss', ] MAX_EVAL_METRICS = ['average_precision', 'ssim', 'accuracy', 'bleu_score'] @@ -129,7 +128,7 @@ def get_index_that_reaches_target(workload_df, op = operator.le if is_minimized else operator.ge validation_target_reached = validation_series.apply( lambda x: op(x, validation_target)) - + print(validation_target_reached) target_reached = pd.Series(validation_target_reached[0]) # Remove trials that never reach the target target_reached = target_reached[target_reached.apply(np.any)] From 4441f321a3b9c89991d289d701e827a602cec3f9 Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Tue, 24 Oct 2023 00:15:17 +0000 Subject: [PATCH 03/16] add loss to scoring registry --- scoring/scoring.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scoring/scoring.py b/scoring/scoring.py index 959965d8d..2594efef6 100644 --- a/scoring/scoring.py +++ b/scoring/scoring.py @@ -47,6 +47,7 @@ 'ctc_loss', 'wer', 'l1_loss', + 'loss', ] MAX_EVAL_METRICS = ['average_precision', 'ssim', 'accuracy', 'bleu_score'] From 5fd528a1aae1208147b1323e682efc4c383124f9 Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Tue, 24 Oct 2023 00:16:40 +0000 Subject: [PATCH 04/16] fix index --- scoring/scoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scoring/scoring.py b/scoring/scoring.py index 2594efef6..7d48a42c4 100644 --- a/scoring/scoring.py +++ b/scoring/scoring.py @@ -130,7 +130,7 @@ def get_index_that_reaches_target(workload_df, validation_target_reached = validation_series.apply( lambda x: op(x, validation_target)) print(validation_target_reached) - target_reached = pd.Series(validation_target_reached[0]) + target_reached = pd.Series(validation_target_reached) # Remove trials that never reach the target target_reached = target_reached[target_reached.apply(np.any)] From 0146ecb89a63ed981c4d3b4b5fe6847bd642f587 Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Tue, 24 Oct 2023 00:18:55 +0000 Subject: [PATCH 05/16] add map to max eval metrics --- scoring/scoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scoring/scoring.py b/scoring/scoring.py index 7d48a42c4..c3fa79c52 100644 --- a/scoring/scoring.py +++ b/scoring/scoring.py @@ -50,7 +50,7 @@ 'loss', ] -MAX_EVAL_METRICS = ['average_precision', 'ssim', 'accuracy', 'bleu_score'] +MAX_EVAL_METRICS = ['mean_average_precision', 'ssim', 'accuracy', 'bleu_score'] def generate_eval_cols(metrics): From a415e57d944858b321617e7efb04fb4f22c5360e Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Tue, 24 Oct 2023 00:20:55 +0000 Subject: [PATCH 06/16] add blue to max eval metrics --- scoring/scoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scoring/scoring.py b/scoring/scoring.py index c3fa79c52..6a9974433 100644 --- a/scoring/scoring.py +++ b/scoring/scoring.py @@ -50,7 +50,7 @@ 'loss', ] -MAX_EVAL_METRICS = ['mean_average_precision', 'ssim', 'accuracy', 'bleu_score'] +MAX_EVAL_METRICS = ['mean_average_precision', 'ssim', 'accuracy', 'bleu'] def generate_eval_cols(metrics): From 0d836cfba9fc3ed6b8f9fc3894248920e27dd77f Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Tue, 24 Oct 2023 00:21:47 +0000 Subject: [PATCH 07/16] remove print statement --- scoring/scoring.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scoring/scoring.py b/scoring/scoring.py index 6a9974433..7e52bd08c 100644 --- a/scoring/scoring.py +++ b/scoring/scoring.py @@ -129,7 +129,6 @@ def get_index_that_reaches_target(workload_df, op = operator.le if is_minimized else operator.ge validation_target_reached = validation_series.apply( lambda x: op(x, validation_target)) - print(validation_target_reached) target_reached = pd.Series(validation_target_reached) # Remove trials that never reach the target target_reached = target_reached[target_reached.apply(np.any)] From 8a4f8fbb18ddfee676713cb6abfebfe5f32333e9 Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Tue, 24 Oct 2023 00:40:58 +0000 Subject: [PATCH 08/16] debugging --- scoring/scoring.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scoring/scoring.py b/scoring/scoring.py index 7e52bd08c..3c18cbcb3 100644 --- a/scoring/scoring.py +++ b/scoring/scoring.py @@ -355,6 +355,7 @@ def plot_performance_profiles(perf_df, Returns: None. If a valid save_dir is provided, save both the plot and perf_df. """ + print(perf_df) fig = perf_df.T.plot(figsize=figsize) df_col_display = f'log10({df_col})' if scale == 'log' else df_col fig.set_xlabel( From d70f4321885ad42a5c05606d55fa6547a2c0db56 Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Tue, 24 Oct 2023 00:45:01 +0000 Subject: [PATCH 09/16] df --- scoring/scoring.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scoring/scoring.py b/scoring/scoring.py index 3c18cbcb3..9b62f985a 100644 --- a/scoring/scoring.py +++ b/scoring/scoring.py @@ -34,6 +34,7 @@ import matplotlib.pyplot as plt import numpy as np import pandas as pd +from tabulate import tabulate import algorithmic_efficiency.workloads.workloads as workloads_registry @@ -355,7 +356,7 @@ def plot_performance_profiles(perf_df, Returns: None. If a valid save_dir is provided, save both the plot and perf_df. """ - print(perf_df) + print(tabulate(pef_df, headers='keys', tablefmt='psql')) fig = perf_df.T.plot(figsize=figsize) df_col_display = f'log10({df_col})' if scale == 'log' else df_col fig.set_xlabel( From 03ad1df9bf0a80b396c9ff1002dd82ee201da6ea Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Tue, 24 Oct 2023 00:47:24 +0000 Subject: [PATCH 10/16] fix --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index a7ce5ebb2..4c2d9e6d3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,7 @@ install_requires = absl-py==1.4.0 numpy>=1.23 pandas>=2.0.1 + tabulate==0.9.0 tensorflow==2.12.0 tensorflow-datasets==4.9.2 tensorflow-probability==0.20.0 From 8a937ee02819f6a455f5aa6f31fbafbd84779934 Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Tue, 24 Oct 2023 00:48:34 +0000 Subject: [PATCH 11/16] fix --- scoring/scoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scoring/scoring.py b/scoring/scoring.py index 9b62f985a..b51be9bf5 100644 --- a/scoring/scoring.py +++ b/scoring/scoring.py @@ -356,7 +356,7 @@ def plot_performance_profiles(perf_df, Returns: None. If a valid save_dir is provided, save both the plot and perf_df. """ - print(tabulate(pef_df, headers='keys', tablefmt='psql')) + print(tabulate(perf_df, headers='keys', tablefmt='psql')) fig = perf_df.T.plot(figsize=figsize) df_col_display = f'log10({df_col})' if scale == 'log' else df_col fig.set_xlabel( From 87e0762b1d5ee824da1e3be8c6631479e9ce1e68 Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Tue, 24 Oct 2023 01:00:26 +0000 Subject: [PATCH 12/16] verbosity --- scoring/score_submission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scoring/score_submission.py b/scoring/score_submission.py index 42a605dac..c25e6f960 100644 --- a/scoring/score_submission.py +++ b/scoring/score_submission.py @@ -32,7 +32,7 @@ def main(_): reference_submission_tag=None, num_points=100, scale='linear', - verbosity=0) + verbosity=1) if not os.path.exists(FLAGS.output_dir): os.mkdir(FLAGS.output_dir) scoring.plot_performance_profiles( From 67232324d964e5f418dfc7fc5171d10a1993d860 Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Tue, 24 Oct 2023 01:14:38 +0000 Subject: [PATCH 13/16] fix --- scoring/score_submission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scoring/score_submission.py b/scoring/score_submission.py index c25e6f960..4523966eb 100644 --- a/scoring/score_submission.py +++ b/scoring/score_submission.py @@ -32,7 +32,7 @@ def main(_): reference_submission_tag=None, num_points=100, scale='linear', - verbosity=1) + ) if not os.path.exists(FLAGS.output_dir): os.mkdir(FLAGS.output_dir) scoring.plot_performance_profiles( From e9d3c7d7ff0d78bdd942db6fcaf75bf5c116eb22 Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Wed, 25 Oct 2023 20:35:46 +0000 Subject: [PATCH 14/16] debugging print statements --- scoring/score_submission.py | 1 + scoring/scoring.py | 23 +++++++++++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/scoring/score_submission.py b/scoring/score_submission.py index 4523966eb..b51886cf5 100644 --- a/scoring/score_submission.py +++ b/scoring/score_submission.py @@ -21,6 +21,7 @@ def main(_): df = scoring_utils.get_experiment_df(FLAGS.experiment_path) + print(df) results = { FLAGS.submission_tag: df, } diff --git a/scoring/scoring.py b/scoring/scoring.py index b51be9bf5..fcf813b8b 100644 --- a/scoring/scoring.py +++ b/scoring/scoring.py @@ -140,7 +140,10 @@ def get_index_that_reaches_target(workload_df, return -1, -1 else: index_reached = target_reached.apply(np.argmax) + print(index_reached) trial = index_reached.idxmin() + print(trial) + print(index_reached) return trial, index_reached[trial] @@ -165,6 +168,7 @@ def get_times_for_submission(submission, for workload, group in submission.groupby('workload'): workload_name = re.match(WORKLOAD_NAME_PATTERN, workload).group(1) + print(workload_name) framework = re.match(WORKLOAD_NAME_PATTERN, workload).group(2) workload_metadata = WORKLOADS[workload_name] @@ -268,18 +272,19 @@ def compute_performance_profiles(results, if verbosity > 0: print(f'\n`{time_col}` to reach target normalized to best:') - with pd.option_context('display.max_rows', - None, - 'display.max_columns', - None, - 'display.width', - 1000): - print(df) + # with pd.option_context('display.max_rows', + # None, + # 'display.max_columns', + # None, + # 'display.width', + # 1000): + # print(df) # If no max_tau is supplied, choose the value of tau that would plot all non # inf or nan data. if max_tau is None: max_tau = df.replace(float('inf'), -1).replace(np.nan, -1).values.max() + print(f"MAX TAU: {max_tau}") if scale == 'linear': points = np.linspace(min_tau, max_tau, num=num_points) @@ -356,7 +361,9 @@ def plot_performance_profiles(perf_df, Returns: None. If a valid save_dir is provided, save both the plot and perf_df. """ - print(tabulate(perf_df, headers='keys', tablefmt='psql')) + print("PERF DF") + print(perf_df.columns) + print(perf_df.T) fig = perf_df.T.plot(figsize=figsize) df_col_display = f'log10({df_col})' if scale == 'log' else df_col fig.set_xlabel( From 4df526b7b293719855be42bddd98db6be20960a0 Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Mon, 30 Oct 2023 20:17:33 +0000 Subject: [PATCH 15/16] remove debugging print statements --- scoring/score_submission.py | 3 +-- scoring/scoring.py | 22 +++++++--------------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/scoring/score_submission.py b/scoring/score_submission.py index b51886cf5..42a605dac 100644 --- a/scoring/score_submission.py +++ b/scoring/score_submission.py @@ -21,7 +21,6 @@ def main(_): df = scoring_utils.get_experiment_df(FLAGS.experiment_path) - print(df) results = { FLAGS.submission_tag: df, } @@ -33,7 +32,7 @@ def main(_): reference_submission_tag=None, num_points=100, scale='linear', - ) + verbosity=0) if not os.path.exists(FLAGS.output_dir): os.mkdir(FLAGS.output_dir) scoring.plot_performance_profiles( diff --git a/scoring/scoring.py b/scoring/scoring.py index fcf813b8b..0c076bea4 100644 --- a/scoring/scoring.py +++ b/scoring/scoring.py @@ -140,10 +140,7 @@ def get_index_that_reaches_target(workload_df, return -1, -1 else: index_reached = target_reached.apply(np.argmax) - print(index_reached) trial = index_reached.idxmin() - print(trial) - print(index_reached) return trial, index_reached[trial] @@ -168,7 +165,6 @@ def get_times_for_submission(submission, for workload, group in submission.groupby('workload'): workload_name = re.match(WORKLOAD_NAME_PATTERN, workload).group(1) - print(workload_name) framework = re.match(WORKLOAD_NAME_PATTERN, workload).group(2) workload_metadata = WORKLOADS[workload_name] @@ -272,19 +268,18 @@ def compute_performance_profiles(results, if verbosity > 0: print(f'\n`{time_col}` to reach target normalized to best:') - # with pd.option_context('display.max_rows', - # None, - # 'display.max_columns', - # None, - # 'display.width', - # 1000): - # print(df) + with pd.option_context('display.max_rows', + None, + 'display.max_columns', + None, + 'display.width', + 1000): + print(df) # If no max_tau is supplied, choose the value of tau that would plot all non # inf or nan data. if max_tau is None: max_tau = df.replace(float('inf'), -1).replace(np.nan, -1).values.max() - print(f"MAX TAU: {max_tau}") if scale == 'linear': points = np.linspace(min_tau, max_tau, num=num_points) @@ -361,9 +356,6 @@ def plot_performance_profiles(perf_df, Returns: None. If a valid save_dir is provided, save both the plot and perf_df. """ - print("PERF DF") - print(perf_df.columns) - print(perf_df.T) fig = perf_df.T.plot(figsize=figsize) df_col_display = f'log10({df_col})' if scale == 'log' else df_col fig.set_xlabel( From 1cb02ca0c2e31165241136b952b4c266f49c039e Mon Sep 17 00:00:00 2001 From: priyakasimbeg Date: Thu, 2 Nov 2023 15:19:55 -0700 Subject: [PATCH 16/16] remove tabulate import --- scoring/scoring.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scoring/scoring.py b/scoring/scoring.py index 0c076bea4..7e52bd08c 100644 --- a/scoring/scoring.py +++ b/scoring/scoring.py @@ -34,7 +34,6 @@ import matplotlib.pyplot as plt import numpy as np import pandas as pd -from tabulate import tabulate import algorithmic_efficiency.workloads.workloads as workloads_registry