feat: AIPerf plot configuration system

src/aiperf/cli.py

@@ -40,6 +40,8 @@ def plot(
     paths: list[str] | None = None,
     output: str | None = None,
     theme: str = "light",
+    config: str | None = None,
+    verbose: bool = False,
 ) -> None:
     """Generate PNG visualizations from AIPerf profiling data.
 
@@ -48,12 +50,28 @@ def plot(
     whether to generate multi-run comparison plots or single-run time series plots
     based on the directory structure.
 
+    On first run, automatically creates ~/.aiperf/plot_config.yaml which you can edit to
+    customize plots, including experiment classification (baseline vs treatment runs).
+    Use --config to specify a different config file.
+
+    Examples:
+        # Generate plots (auto-creates ~/.aiperf/plot_config.yaml on first run)
+        aiperf plot
+
+        # Use custom config
+        aiperf plot --config my_plots.yaml
+
+        # Show detailed error tracebacks
+        aiperf plot --verbose
+
     Args:
         paths: Paths to profiling run directories. Defaults to ./artifacts if not specified.
         output: Directory to save generated plots. Defaults to <first_path>/plots if not specified.
         theme: Plot theme to use: 'light' (white background) or 'dark' (dark background). Defaults to 'light'.
+        config: Path to custom plot configuration YAML file. If not specified, auto-creates and uses ~/.aiperf/plot_config.yaml.
+        verbose: Show detailed error tracebacks in console (errors are always logged to <output_dir>/aiperf_plot.log).
     """
-    with exit_on_error(title="Error Running Plot Command"):
+    with exit_on_error(title="Error Running Plot Command", show_traceback=verbose):
         from aiperf.plot.cli_runner import run_plot_controller
 
-        run_plot_controller(paths, output, theme=theme)
+        run_plot_controller(paths, output, theme=theme, config=config, verbose=verbose)

src/aiperf/cli_utils.py

@@ -68,6 +68,7 @@ class exit_on_error(AbstractContextManager):
         text_color: The text color to use.
         title: The title of the error.
         exit_code: The exit code to use.
+        show_traceback: Whether to show the full exception traceback. Defaults to True.
     """
 
     def __init__(
@@ -77,12 +78,14 @@ def __init__(
         text_color: "StyleType | None" = None,
         title: str = "Error",
         exit_code: int = 1,
+        show_traceback: bool = True,
     ):
         self.message: RenderableType = message
         self.text_color: StyleType | None = text_color
         self.title: str = title
         self.exit_code: int = exit_code
         self.exceptions: tuple[type[BaseException], ...] = exceptions
+        self.show_traceback: bool = show_traceback
 
     def __enter__(self):
         return self
@@ -98,13 +101,17 @@ def __exit__(self, exc_type, exc_value, traceback):
             from rich.console import Console
 
             console = Console()
-            console.print_exception(
-                show_locals=True,
-                max_frames=10,
-                word_wrap=True,
-                width=console.width,
-            )
-            console.file.flush()
+
+            # Only show full traceback if requested
+            if self.show_traceback:
+                console.print_exception(
+                    show_locals=True,
+                    max_frames=10,
+                    word_wrap=True,
+                    width=console.width,
+                )
+                console.file.flush()
+
             message = (
                 self.message.format(e=exc_value)
                 if isinstance(self.message, str)

src/aiperf/common/enums/__init__.py

@@ -67,6 +67,7 @@
     MetricValueType,
     MetricValueTypeInfo,
     MetricValueTypeVarT,
+    PlotMetricDirection,
     PowerMetricUnit,
     PowerMetricUnitInfo,
     TemperatureMetricUnit,
@@ -153,6 +154,7 @@
     "MetricValueTypeInfo",
     "MetricValueTypeVarT",
     "ModelSelectionStrategy",
+    "PlotMetricDirection",
     "PowerMetricUnit",
     "PowerMetricUnitInfo",
     "PromptSource",

src/aiperf/common/enums/metric_enums.py

@@ -414,6 +414,16 @@ class MetricType(CaseInsensitiveStrEnum):
     Examples: request throughput, output token throughput, etc."""
 
 
+class PlotMetricDirection(CaseInsensitiveStrEnum):
+    """Direction indicating whether higher or lower metric values are better for plotting purposes."""
+
+    HIGHER = "higher"
+    """Higher values are better (e.g., throughput, accuracy)."""
+
+    LOWER = "lower"
+    """Lower values are better (e.g., latency, error rate)."""
+
+
 class MetricValueTypeInfo(BasePydanticEnumInfo):
     """Information about a metric value type."""
 

src/aiperf/plot/__init__.py

@@ -12,6 +12,10 @@
 from aiperf.plot.cli_runner import (
     run_plot_controller,
 )
+from aiperf.plot.config import (
+    PlotConfig,
+    logger,
+)
 from aiperf.plot.constants import (
     ALL_STAT_KEYS,
     AVAILABLE_STATS,
@@ -23,6 +27,7 @@
     DEFAULT_PLOT_HEIGHT,
     DEFAULT_PLOT_WIDTH,
     DEFAULT_PNG_OUTPUT_DIR,
+    DERIVED_METRIC_DIRECTIONS,
     LIGHT_THEME_COLORS,
     NON_METRIC_KEYS,
     NVIDIA_BORDER_DARK,
@@ -35,6 +40,7 @@
     NVIDIA_GREEN,
     NVIDIA_TEXT_LIGHT,
     NVIDIA_WHITE,
+    OUTLIER_RED,
     PLOT_FONT_FAMILY,
     PLOT_LOG_FILE,
     PROFILE_EXPORT_AIPERF_JSON,
@@ -49,6 +55,7 @@
     DataLoader,
     DataSource,
     DerivedMetricCalculator,
+    ExperimentClassificationConfig,
     MetricSpec,
     ModeDetector,
     PlotGenerator,
@@ -66,6 +73,7 @@
     auto_select_label_by,
     calculate_rolling_percentiles,
     calculate_throughput_events,
+    detect_directional_outliers,
     detect_swept_parameters,
     flatten_config,
     get_nvidia_color_scheme,
@@ -98,11 +106,16 @@
     TimeSliceHandler,
 )
 from aiperf.plot.logging import (
+    setup_console_only_logging,
     setup_plot_logging,
 )
 from aiperf.plot.metric_names import (
+    get_aggregated_metrics,
     get_all_metric_display_names,
+    get_gpu_metrics,
     get_metric_display_name,
+    get_request_metrics,
+    get_timeslice_metrics,
 )
 from aiperf.plot.plot_controller import (
     PlotController,
@@ -125,11 +138,13 @@
     "DEFAULT_PLOT_HEIGHT",
     "DEFAULT_PLOT_WIDTH",
     "DEFAULT_PNG_OUTPUT_DIR",
+    "DERIVED_METRIC_DIRECTIONS",
     "DataLoadError",
     "DataLoader",
     "DataSource",
     "DerivedMetricCalculator",
     "DualAxisHandler",
+    "ExperimentClassificationConfig",
     "HistogramHandler",
     "LIGHT_THEME_COLORS",
     "MetricSpec",
@@ -147,13 +162,15 @@
     "NVIDIA_GREEN",
     "NVIDIA_TEXT_LIGHT",
     "NVIDIA_WHITE",
+    "OUTLIER_RED",
     "PLOT_FONT_FAMILY",
     "PLOT_LOG_FILE",
     "PROFILE_EXPORT_AIPERF_JSON",
     "PROFILE_EXPORT_GPU_TELEMETRY_JSONL",
     "PROFILE_EXPORT_JSONL",
     "PROFILE_EXPORT_TIMESLICES_CSV",
     "ParetoHandler",
+    "PlotConfig",
     "PlotController",
     "PlotError",
     "PlotGenerationError",
@@ -179,14 +196,21 @@
     "auto_select_label_by",
     "calculate_rolling_percentiles",
     "calculate_throughput_events",
+    "detect_directional_outliers",
     "detect_swept_parameters",
     "flatten_config",
+    "get_aggregated_metrics",
     "get_all_metric_display_names",
+    "get_gpu_metrics",
     "get_metric_display_name",
     "get_nvidia_color_scheme",
+    "get_request_metrics",
+    "get_timeslice_metrics",
+    "logger",
     "prepare_request_timeseries",
     "prepare_timeslice_metrics",
     "run_plot_controller",
+    "setup_console_only_logging",
     "setup_plot_logging",
     "validate_request_uniformity",
 ]

src/aiperf/plot/cli_runner.py

@@ -4,7 +4,7 @@
 
 from pathlib import Path
 
-from aiperf.plot.constants import PlotMode, PlotTheme
+from aiperf.plot.constants import PLOT_LOG_FILE, PlotMode, PlotTheme
 from aiperf.plot.plot_controller import PlotController
 
 
@@ -13,6 +13,8 @@ def run_plot_controller(
     output: str | None = None,
     mode: PlotMode | str = PlotMode.PNG,
     theme: PlotTheme | str = PlotTheme.LIGHT,
+    config: str | None = None,
+    verbose: bool = False,
 ) -> None:
     """Generate plots from AIPerf profiling data.
 
@@ -21,6 +23,8 @@ def run_plot_controller(
         output: Directory to save generated plots. Defaults to <first_path>/plots if not specified.
         mode: Output mode for plots. Defaults to PNG.
         theme: Plot theme to use (LIGHT or DARK). Defaults to LIGHT.
+        config: Path to custom plot configuration YAML file. If not specified, uses default config.
+        verbose: Show detailed error tracebacks in console.
     """
     input_paths = paths or ["./artifacts"]
     input_paths = [Path(p) for p in input_paths]
@@ -32,14 +36,19 @@ def run_plot_controller(
     if isinstance(theme, str):
         theme = PlotTheme(theme.lower())
 
+    config_path = Path(config) if config else None
+
     controller = PlotController(
         paths=input_paths,
         output_dir=output_dir,
         mode=mode,
         theme=theme,
+        config_path=config_path,
+        verbose=verbose,
     )
 
     generated_files = controller.run()
 
     print(f"\nGenerated {len(generated_files)} plots")
     print(f"Saved to: {output_dir}")
+    print(f"Logs: {output_dir / PLOT_LOG_FILE}")