diff --git a/.circleci/config.yml b/.circleci/config.yml index 90a8a9abb..32ad29cdf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -341,12 +341,12 @@ jobs: - run_nvidia_smi - benchmark_layers_integration_test: device: "cuda" - layers: "groupnorm gsm_groupnorm instancenorm gsm_instancenorm layernorm gsm_layernorm" + layers: "groupnorm gsm_groupnorm instancenorm gsm_instancenorm layernorm gsm_layernorm dpmha" runtime_ratio_threshold: "2.5" memory_ratio_threshold: "1.6" - benchmark_layers_integration_test: device: "cuda" - layers: linear gsm_linear + layers: "linear gsm_linear" runtime_ratio_threshold: "3.6" memory_ratio_threshold: "13.0" - benchmark_layers_integration_test: diff --git a/benchmarks/utils.py b/benchmarks/utils.py index a21ddeddb..8de6a33d2 100644 --- a/benchmarks/utils.py +++ b/benchmarks/utils.py @@ -220,12 +220,13 @@ def generate_report(path_to_results: str, save_path: str, format: str) -> None: ) def add_ratio(df, metric, variant): - if variant in df.columns.get_level_values("variant"): - df[(metric, f"{variant}/control")] = ( - df.loc[:, (metric, variant)] / df.loc[:, (metric, "control")] - ) - else: - df[(metric, f"{variant}/control")] = np.nan + if variant not in df.columns.get_level_values("variant"): + for ametric in df.columns.get_level_values(0): + df[(ametric, variant)] = np.nan + + df[(metric, f"{variant}/control")] = ( + df.loc[:, (metric, variant)] / df.loc[:, (metric, "control")] + ) if "control" in results["variant"].tolist(): add_ratio(pivot, "runtime", "dp")