From f73997e527bc3113aa51c772ec3dbfb664332736 Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Sat, 20 Sep 2025 23:28:04 -0400
Subject: [PATCH 01/40] Begin cleaning up metadata type

---
 cpp/bindings/main.cpp          |   2 +
 python/evalio/__init__.py      |   2 -
 python/evalio/datasets/base.py |   3 +
 python/evalio/types.py         | 174 +++++++++++++++++++++++++++++++--
 4 files changed, 173 insertions(+), 8 deletions(-)

diff --git a/cpp/bindings/main.cpp b/cpp/bindings/main.cpp
index ec466dce..8397dd06 100644
--- a/cpp/bindings/main.cpp
+++ b/cpp/bindings/main.cpp
@@ -8,6 +8,8 @@
 namespace nb = nanobind;
 
 NB_MODULE(_cpp, m) {
+  nb::set_leak_warnings(false);
+
   m.def(
     "abi_tag",
     []() { return nb::detail::abi_tag(); },
diff --git a/python/evalio/__init__.py b/python/evalio/__init__.py
index a86dd76a..3acdb163 100644
--- a/python/evalio/__init__.py
+++ b/python/evalio/__init__.py
@@ -6,8 +6,6 @@
 from . import _cpp, datasets, pipelines, stats, types, utils
 from ._cpp import abi_tag as _abi_tag
 
-Param = bool | int | float | str
-
 
 # remove false nanobind reference leak warnings
 # https://github.com/wjakob/nanobind/discussions/13
diff --git a/python/evalio/datasets/base.py b/python/evalio/datasets/base.py
index 6ce93a57..467cc26e 100644
--- a/python/evalio/datasets/base.py
+++ b/python/evalio/datasets/base.py
@@ -6,6 +6,7 @@
 
 from evalio.types import (
     SE3,
+    GroundTruth,
     ImuMeasurement,
     ImuParams,
     LidarMeasurement,
@@ -222,6 +223,8 @@ def ground_truth(self) -> Trajectory:
             gt_o_T_gt_i = gt_traj.poses[i]
             gt_traj.poses[i] = gt_o_T_gt_i * gt_T_imu
 
+        gt_traj.metadata = GroundTruth(sequence=self.full_name)
+
         return gt_traj
 
     def _fail_not_downloaded(self):
diff --git a/python/evalio/types.py b/python/evalio/types.py
index abe4ff4d..b6458a59 100644
--- a/python/evalio/types.py
+++ b/python/evalio/types.py
@@ -1,8 +1,10 @@
 import csv
 from dataclasses import dataclass, field
+from enum import Enum
 from pathlib import Path
 from typing import Optional
 
+from evalio.utils import print_warning
 import numpy as np
 import yaml
 
@@ -18,6 +20,156 @@
     Stamp,
 )
 
+Param = bool | int | float | str
+
+
+@dataclass(kw_only=True)
+class GroundTruth:
+    sequence: str
+    """Dataset used to run the experiment."""
+
+    @staticmethod
+    def from_yaml(yaml_str: str) -> Optional["GroundTruth"]:
+        """Create a GroundTruth object from a YAML string.
+
+        Args:
+            yaml_str (str): YAML string to parse.
+        """
+        data = yaml.safe_load(yaml_str)
+
+        gt: Optional[bool] = data.pop("gt", None)
+        sequence = data.pop("sequence", None)
+
+        if gt is None or not gt or sequence is None:
+            return None
+
+        return GroundTruth(sequence=sequence)
+
+    def to_yaml(self) -> str:
+        """Convert the GroundTruth object to a YAML string.
+
+        Returns:
+            str: YAML string representation of the GroundTruth object.
+        """
+        data = {
+            "gt": True,
+            "sequence": self.sequence,
+        }
+
+        return yaml.dump(data)
+
+
+class ExperimentStatus(Enum):
+    Complete = "complete"
+    Fail = "fail"
+    Started = "started"
+    NeverRan = "never_ran"
+
+
+@dataclass(kw_only=True)
+class Experiment:
+    name: str
+    """Name of the experiment."""
+    status: ExperimentStatus
+    """Status of the experiment, e.g. "success", "failure", etc."""
+    sequence: str
+    """Dataset used to run the experiment."""
+    sequence_length: int
+    """Length of the sequence, if set"""
+    pipeline: str
+    """Pipeline used to generate the trajectory."""
+    pipeline_version: str
+    """Version of the pipeline used."""
+    pipeline_params: dict[str, Param] = field(default_factory=dict)
+    """Parameters used for the pipeline."""
+    total_elapsed: Optional[float] = None
+    """Total time taken for the experiment, as a string."""
+    max_step_elapsed: Optional[float] = None
+    """Maximum time taken for a single step in the experiment, as a string."""
+
+    @staticmethod
+    def from_yaml(yaml_str: str) -> Optional["Experiment"]:
+        """Create an Experiment object from a YAML string.
+
+        Args:
+            yaml_str (str): YAML string to parse.
+        """
+        data = yaml.safe_load(yaml_str)
+
+        name = data.pop("name", None)
+        pipeline = data.pop("pipeline", None)
+        pipeline_version = data.pop("pipeline_version", None)
+        pipeline_params = data.pop("pipeline_params", None)
+        sequence = data.pop("sequence", None)
+        sequence_length = data.pop("sequence_length", None)
+
+        if (
+            name is None
+            or sequence is None
+            or sequence_length is None
+            or pipeline is None
+            or pipeline_version is None
+            or pipeline_params is None
+        ):
+            return None
+
+        total_elapsed = data.pop("total_elapsed", None)
+        max_step_elapsed = data.pop("max_step_elapsed", None)
+
+        if "status" in data:
+            status = ExperimentStatus(data.pop("status"))
+        else:
+            status = ExperimentStatus.Started
+
+        if len(data) > 0:
+            # Unknown fields
+            print_warning(
+                f"Experiment.from_yaml: Unknown fields in YAML: {list(data.keys())}"
+            )
+
+        return Experiment(
+            name=name,
+            sequence=sequence,
+            sequence_length=sequence_length,
+            pipeline=pipeline,
+            pipeline_version=pipeline_version,
+            pipeline_params=pipeline_params,
+            status=status,
+            total_elapsed=total_elapsed,
+            max_step_elapsed=max_step_elapsed,
+        )
+
+    def to_yaml(self) -> str:
+        """Convert the Experiment object to a YAML string.
+
+        Returns:
+            str: YAML string representation of the Experiment object.
+        """
+        data: dict[str, Param | dict[str, Param]] = {
+            "name": self.name,
+            "sequence": self.sequence,
+            "sequence_length": self.sequence_length,
+            "pipeline": self.pipeline,
+            "pipeline_params": self.pipeline_params,
+        }
+        if self.status in [ExperimentStatus.Complete, ExperimentStatus.Fail]:
+            data["status"] = self.status.value
+        if self.total_elapsed is not None:
+            data["total_elapsed"] = self.total_elapsed
+        if self.max_step_elapsed is not None:
+            data["max_step_elapsed"] = self.max_step_elapsed
+
+        return yaml.dump(data)
+
+
+def _parse_metadata(yaml_str: str) -> Optional[GroundTruth | Experiment]:
+    if "gt:" in yaml_str:
+        return GroundTruth.from_yaml(yaml_str)
+    elif "name:" in yaml_str:
+        return Experiment.from_yaml(yaml_str)
+    else:
+        return None
+
 
 @dataclass(kw_only=True)
 class Trajectory:
@@ -25,7 +177,7 @@ class Trajectory:
     """List of timestamps for each pose."""
     poses: list[SE3]
     """List of poses, in the same order as the timestamps."""
-    metadata: dict[str, bool | int | float | str] = field(default_factory=dict)
+    metadata: Optional[GroundTruth | Experiment] = None
     """Metadata associated with the trajectory, such as the dataset name or other information."""
 
     def __post_init__(self):
@@ -125,7 +277,7 @@ def from_tum(path: Path) -> "Trajectory":
         return Trajectory.from_csv(path, ["sec", "x", "y", "z", "qx", "qy", "qz", "qw"])
 
     @staticmethod
-    def from_experiment(path: Path) -> "Trajectory":
+    def from_experiment(path: Path) -> Optional["Trajectory"]:
         """Load a saved experiment trajectory from file.
 
         Works identically to [from_tum][evalio.types.Trajectory.from_tum], but also loads metadata from the file.
@@ -137,12 +289,15 @@ def from_experiment(path: Path) -> "Trajectory":
             Trajectory: Loaded trajectory with metadata, stamps, and poses.
         """
         with open(path) as file:
-            metadata_filter = filter(lambda row: row[0] == "#", file)
+            metadata_filter = filter(
+                lambda row: row[0] == "#" and not row.startswith("# timestamp,"), file
+            )
             metadata_list = [row[1:].strip() for row in metadata_filter]
-            # remove the header row
-            metadata_list.pop(-1)
+            if len(metadata_list) == 0:
+                return None
+
             metadata_str = "\n".join(metadata_list)
-            metadata = yaml.safe_load(metadata_str)
+            metadata = _parse_metadata(metadata_str)
 
         trajectory = Trajectory.from_csv(
             path,
@@ -153,12 +308,19 @@ def from_experiment(path: Path) -> "Trajectory":
         return trajectory
 
 
+# TODO: Some sort of incremental writer for experiments
+# TODO: Some sort of batch writer for ground truth (can be the same as above)
+
+
 __all__ = [
+    "Experiment",
+    "ExperimentStatus",
     "ImuMeasurement",
     "ImuParams",
     "LidarMeasurement",
     "LidarParams",
     "Duration",
+    "Param",
     "Point",
     "SO3",
     "SE3",

From 97e4ab22461c0ab774430fc4dbc819f0b1131d9c Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Fri, 26 Sep 2025 17:12:30 -0400
Subject: [PATCH 02/40] Write new pipeline and dataset registration methods

---
 python/evalio/datasets/__init__.py  | 13 +++++
 python/evalio/datasets/parser.py    | 71 ++++++++++++++++++++++++
 python/evalio/pipelines/__init__.py |  9 ++++
 python/evalio/pipelines/parser.py   | 83 +++++++++++++++++++++++++++++
 4 files changed, 176 insertions(+)
 create mode 100644 python/evalio/datasets/parser.py
 create mode 100644 python/evalio/pipelines/parser.py

diff --git a/python/evalio/datasets/__init__.py b/python/evalio/datasets/__init__.py
index 11426c72..4455f1af 100644
--- a/python/evalio/datasets/__init__.py
+++ b/python/evalio/datasets/__init__.py
@@ -10,6 +10,14 @@
 from .newer_college_2021 import NewerCollege2021
 from .oxford_spires import OxfordSpires
 
+from .parser import (
+    all_datasets,
+    get_dataset,
+    all_sequences,
+    get_sequence,
+    register_dataset,
+)
+
 __all__ = [
     "get_data_dir",
     "set_data_dir",
@@ -26,4 +34,9 @@
     "OxfordSpires",
     "RawDataIter",
     "RosbagIter",
+    "all_datasets",
+    "get_dataset",
+    "all_sequences",
+    "get_sequence",
+    "register_dataset",
 ]
diff --git a/python/evalio/datasets/parser.py b/python/evalio/datasets/parser.py
new file mode 100644
index 00000000..0519fc86
--- /dev/null
+++ b/python/evalio/datasets/parser.py
@@ -0,0 +1,71 @@
+import importlib
+from inspect import isclass
+from types import ModuleType
+from typing import Optional
+
+from evalio import datasets
+from evalio.datasets.base import Dataset
+
+_DATASETS: set[type[Dataset]] = set()
+
+
+# ------------------------- Handle Registration of Datasets ------------------------- #
+def _is_dataset(obj: object) -> bool:
+    return (
+        isclass(obj) and issubclass(obj, Dataset) and obj.__name__ != Dataset.__name__
+    )
+
+
+def _search_module(module: ModuleType) -> set[type[Dataset]]:
+    return {c for c in module.__dict__.values() if _is_dataset(c)}
+
+
+def register_dataset(
+    dataset: Optional[type[Dataset]] = None,
+    module: Optional[ModuleType | str] = None,
+):
+    global _DATASETS
+
+    if module is not None:
+        if isinstance(module, str):
+            try:
+                module = importlib.import_module(module)
+            except ImportError:
+                raise ValueError(f"Failed to import '{module}'")
+
+        if len(new_ds := _search_module(module)) > 0:
+            _DATASETS.update(new_ds)
+        else:
+            raise ValueError(
+                f"Module {module.__name__} does not contain any datasets or pipelines"
+            )
+
+    if dataset is not None:
+        if _is_dataset(dataset):
+            _DATASETS.add(dataset)
+        else:
+            raise ValueError(f"{dataset} is not a valid Dataset subclass")
+
+
+def all_datasets() -> dict[str, type[Dataset]]:
+    global _DATASETS
+    return {d.dataset_name(): d for d in _DATASETS}
+
+
+def get_dataset(name: str) -> Optional[type[Dataset]]:
+    return all_datasets().get(name, None)
+
+
+def all_sequences() -> dict[str, Dataset]:
+    return {
+        seq.full_name: seq for d in all_datasets().values() for seq in d.sequences()
+    }
+
+
+def get_sequence(name: str) -> Optional[Dataset]:
+    return all_sequences().get(name, None)
+
+
+register_dataset(module=datasets)
+
+# ------------------------- Handle yaml parsing ------------------------- #
diff --git a/python/evalio/pipelines/__init__.py b/python/evalio/pipelines/__init__.py
index c22d0676..145713e5 100644
--- a/python/evalio/pipelines/__init__.py
+++ b/python/evalio/pipelines/__init__.py
@@ -1 +1,10 @@
 from evalio._cpp.pipelines import *  # type: ignore  # noqa: F403
+
+from .parser import register_pipeline, get_pipeline, all_pipelines
+
+
+__all__ = [
+    "all_pipelines",
+    "get_pipeline",
+    "register_pipeline",
+]
diff --git a/python/evalio/pipelines/parser.py b/python/evalio/pipelines/parser.py
new file mode 100644
index 00000000..67929442
--- /dev/null
+++ b/python/evalio/pipelines/parser.py
@@ -0,0 +1,83 @@
+import importlib
+from inspect import isclass
+from types import ModuleType
+from typing import Any, Optional
+
+from evalio import pipelines
+from evalio.pipelines import Pipeline
+
+_PIPELINES: set[type[Pipeline]] = set()
+
+
+# ------------------------- Handle Registration of Pipelines ------------------------- #
+def _is_pipe(obj: Any) -> bool:
+    return (
+        isclass(obj) and issubclass(obj, Pipeline) and obj.__name__ != Pipeline.__name__
+    )
+
+
+def _search_module(module: ModuleType) -> set[type[Pipeline]]:
+    return {c for c in module.__dict__.values() if _is_pipe(c)}
+
+
+def register_pipeline(
+    pipeline: Optional[type[Pipeline]] = None,
+    module: Optional[ModuleType | str] = None,
+):
+    """Add a pipeline or a module containing pipelines to the registry.
+
+    Args:
+        pipeline (Optional[type[Pipeline]], optional): A specific pipeline class to add. Defaults to None.
+        module (Optional[ModuleType  |  str], optional): The module to search for pipelines. Defaults to None.
+
+    Raises:
+        ValueError: If the module does not contain any pipelines.
+        ValueError: If the pipeline is not a valid Pipeline subclass.
+        ValueError: If both module and pipeline are None.
+    """
+    global _PIPELINES
+
+    if module is not None:
+        if isinstance(module, str):
+            try:
+                module = importlib.import_module(module)
+            except ImportError:
+                raise ValueError(f"Failed to import '{module}'")
+
+        if len(new_pipes := _search_module(module)) > 0:
+            _PIPELINES.update(new_pipes)
+        else:
+            raise ValueError(f"Module {module.__name__} does not contain any pipelines")
+
+    if pipeline is not None:
+        if _is_pipe(pipeline):
+            _PIPELINES.add(pipeline)
+        else:
+            raise ValueError(f"{pipeline} is not a valid Pipeline subclass")
+
+
+def all_pipelines() -> dict[str, type[Pipeline]]:
+    """Get all registered pipelines.
+
+    Returns:
+        dict[str, type[Pipeline]]: A dictionary mapping pipeline names to their classes.
+    """
+    global _PIPELINES
+    return {p.name(): p for p in _PIPELINES}
+
+
+def get_pipeline(name: str) -> Optional[type[Pipeline]]:
+    """Get a pipeline class by its name.
+
+    Args:
+        name (str): The name of the pipeline.
+
+    Returns:
+        Optional[type[Pipeline]]: The pipeline class, or None if not found.
+    """
+    return all_pipelines().get(name, None)
+
+
+register_pipeline(module=pipelines)
+
+# ------------------------- Handle yaml parsing ------------------------- #

From 2e2363a20cd14709de9d60bb4e5aa830104619f9 Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Fri, 26 Sep 2025 21:41:42 -0400
Subject: [PATCH 03/40] Small tweaks to clean up changes

---
 python/evalio/cli/parser.py   |   2 +-
 python/evalio/rerun.py        |   4 +-
 python/evalio/stats.py        | 106 +++++++++++++++++++++++++---------
 tests/test_dataset_loading.py |  11 +++-
 tests/test_experiment.py      |  92 +++++++++++++++++++++++++++++
 tests/test_stats.py           |   7 ---
 6 files changed, 182 insertions(+), 40 deletions(-)
 create mode 100644 tests/test_experiment.py

diff --git a/python/evalio/cli/parser.py b/python/evalio/cli/parser.py
index bc6def1b..39701357 100644
--- a/python/evalio/cli/parser.py
+++ b/python/evalio/cli/parser.py
@@ -12,7 +12,7 @@
 import yaml
 
 import evalio
-from evalio import Param
+from evalio.types import Param
 from evalio.datasets import Dataset
 from evalio.pipelines import Pipeline
 from evalio.utils import print_warning
diff --git a/python/evalio/rerun.py b/python/evalio/rerun.py
index 5df2f892..eee577cf 100644
--- a/python/evalio/rerun.py
+++ b/python/evalio/rerun.py
@@ -10,7 +10,7 @@
 from evalio.cli.parser import PipelineBuilder
 from evalio.datasets import Dataset
 from evalio.pipelines import Pipeline
-from evalio.stats import check_overstep
+from evalio.stats import _check_overstep
 from evalio.types import SE3, LidarMeasurement, LidarParams, Point, Stamp, Trajectory
 from evalio.utils import print_warning
 
@@ -204,7 +204,7 @@ def log(
                     gt_index = 0
                     while self.gt.stamps[gt_index] < data.stamp:
                         gt_index += 1
-                    if check_overstep(self.gt.stamps, data.stamp, gt_index):
+                    if _check_overstep(self.gt.stamps, data.stamp, gt_index):
                         gt_index -= 1
                     gt_o_T_imu_0 = self.gt.poses[gt_index]
                     self.gt_o_T_imu_o = gt_o_T_imu_0 * imu_o_T_imu_0.inverse()
diff --git a/python/evalio/stats.py b/python/evalio/stats.py
index 3fab12b7..1b770cf6 100644
--- a/python/evalio/stats.py
+++ b/python/evalio/stats.py
@@ -1,27 +1,19 @@
-from copy import deepcopy
-from dataclasses import dataclass
 from enum import StrEnum, auto
-from typing import cast
-
-import numpy as np
-from numpy.typing import NDArray
 
 from evalio.utils import print_warning
+from .types import Stamp, Trajectory, SE3
 
-from .types import SE3, Stamp, Trajectory
+from dataclasses import dataclass
 
+import numpy as np
 
-def check_overstep(stamps: list[Stamp], s: Stamp, idx: int) -> bool:
-    """Checks if we overshot the closest stamp.
+from typing import Optional, cast
+from numpy.typing import NDArray
 
-    Args:
-        stamps (list[Stamp]): List of stamps
-        s (Stamp): Stamp we want to find the closest to
-        idx (int): Index of the closest stamp
+from copy import deepcopy
 
-    Returns:
-        bool: True if we overshot the closest stamp (ie it's idx - 1), False if it's good (ie it's idx)
-    """
+
+def _check_overstep(stamps: list[Stamp], s: Stamp, idx: int) -> bool:
     return abs((stamps[idx - 1] - s).to_sec()) < abs((stamps[idx] - s).to_sec())
 
 
@@ -36,6 +28,15 @@ class MetricKind(StrEnum):
     """Sqrt of Sum of squared errors"""
 
 
+class WindowKind(StrEnum):
+    """Simple enum to define whether the window computed should be based on distance or time."""
+
+    distance = auto()
+    """Window based on distance"""
+    time = auto()
+    """Window based on time"""
+
+
 @dataclass(kw_only=True)
 class Metric:
     """Simple dataclass to hold the resulting metrics. Likely output from [Error][evalio.stats.Error]."""
@@ -154,7 +155,7 @@ def align_stamps(traj1: Trajectory, traj2: Trajectory):
     first_pose_idx = 0
     while traj1.stamps[first_pose_idx] < traj2.stamps[0]:
         first_pose_idx += 1
-    if check_overstep(traj1.stamps, traj2.stamps[0], first_pose_idx):
+    if _check_overstep(traj1.stamps, traj2.stamps[0], first_pose_idx):
         first_pose_idx -= 1
     traj1.stamps = traj1.stamps[first_pose_idx:]
     traj1.poses = traj1.poses[first_pose_idx:]
@@ -163,7 +164,7 @@ def align_stamps(traj1: Trajectory, traj2: Trajectory):
     first_pose_idx = 0
     while traj2.stamps[first_pose_idx] < traj1.stamps[0]:
         first_pose_idx += 1
-    if check_overstep(traj2.stamps, traj1.stamps[0], first_pose_idx):
+    if _check_overstep(traj2.stamps, traj1.stamps[0], first_pose_idx):
         first_pose_idx -= 1
     traj2.stamps = traj2.stamps[first_pose_idx:]
     traj2.poses = traj2.poses[first_pose_idx:]
@@ -186,7 +187,7 @@ def align_stamps(traj1: Trajectory, traj2: Trajectory):
             traj1_idx += 1
 
         # go back one if we overshot
-        if check_overstep(traj1.stamps, stamp, traj1_idx):
+        if _check_overstep(traj1.stamps, stamp, traj1_idx):
             traj1_idx -= 1
 
         traj1_stamps.append(traj1.stamps[traj1_idx])
@@ -220,9 +221,9 @@ def _compute_metric(gts: list[SE3], poses: list[SE3]) -> Error:
     error_r = np.zeros(len(gts))
     for i, (gt, pose) in enumerate(zip(gts, poses)):
         delta = gt.inverse() * pose
-        error_t[i] = np.sqrt(delta.trans @ delta.trans)  # type: ignore
+        error_t[i] = np.sqrt(delta.trans @ delta.trans)
         r_diff = delta.rot.log()
-        error_r[i] = np.sqrt(r_diff @ r_diff) * 180 / np.pi  # type: ignore
+        error_r[i] = np.sqrt(r_diff @ r_diff) * 180 / np.pi
 
     return Error(rot=error_r, trans=error_t)
 
@@ -230,6 +231,8 @@ def _compute_metric(gts: list[SE3], poses: list[SE3]) -> Error:
 def _check_aligned(traj: Trajectory, gt: Trajectory) -> bool:
     """Check if the two trajectories are aligned.
 
+    This is done by checking if the first poses are identical, and if there's the same number of poses in both trajectories.
+
     Args:
         traj (Trajectory): One of the trajectories
         gt (Trajectory): The other trajectory
@@ -239,9 +242,12 @@ def _check_aligned(traj: Trajectory, gt: Trajectory) -> bool:
     """
     # Check if the two trajectories are aligned
     delta = gt.poses[0].inverse() * traj.poses[0]
-    t = delta.trans
     r = delta.rot.log()
-    return len(traj.stamps) == len(gt.stamps) and (t @ t < 1e-6) and (r @ r < 1e-6)  # type: ignore
+    return bool(
+        len(traj.stamps) == len(gt.stamps)
+        and (delta.trans @ delta.trans < 1e-6)
+        and (r @ r < 1e-6)
+    )
 
 
 def ate(traj: Trajectory, gt: Trajectory) -> Error:
@@ -264,7 +270,12 @@ def ate(traj: Trajectory, gt: Trajectory) -> Error:
     return _compute_metric(gt.poses, traj.poses)
 
 
-def rte(traj: Trajectory, gt: Trajectory, window: int = 100) -> Error:
+def rte(
+    traj: Trajectory,
+    gt: Trajectory,
+    kind: WindowKind = WindowKind.time,
+    window: Optional[float | int] = None,
+) -> Error:
     """Compute the Relative Trajectory Error (RTE) between two trajectories.
 
     Will check if the two trajectories are aligned and if not, will align them.
@@ -273,7 +284,8 @@ def rte(traj: Trajectory, gt: Trajectory, window: int = 100) -> Error:
     Args:
         traj (Trajectory): One of the trajectories
         gt (Trajectory): The other trajectory
-        window (int, optional): Window size for the RTE. Defaults to 100.
+        kind (WindowKind, optional): The kind of window to use for the RTE. Defaults to WindowKind.time.
+        window (int | float, optional): Window size for the RTE. If window kind is distance, defaults to 10m. If time, defaults to 100 scans.
 
     Returns:
         Error: The computed error
@@ -281,6 +293,13 @@ def rte(traj: Trajectory, gt: Trajectory, window: int = 100) -> Error:
     if not _check_aligned(traj, gt):
         traj, gt = align(traj, gt)
 
+    if window is None:
+        match kind:
+            case WindowKind.distance:
+                window = 10
+            case WindowKind.time:
+                window = 200
+
     if window <= 0:
         raise ValueError("Window size must be positive")
 
@@ -290,9 +309,40 @@ def rte(traj: Trajectory, gt: Trajectory, window: int = 100) -> Error:
 
     window_deltas_poses: list[SE3] = []
     window_deltas_gts: list[SE3] = []
-    for i in range(len(gt) - window):
-        window_deltas_poses.append(traj.poses[i].inverse() * traj.poses[i + window])
-        window_deltas_gts.append(gt.poses[i].inverse() * gt.poses[i + window])
+
+    if kind == WindowKind.time:
+        assert isinstance(window, int), (
+            "Window size must be an integer for time-based RTE"
+        )
+        for i in range(len(gt) - window):
+            window_deltas_poses.append(traj.poses[i].inverse() * traj.poses[i + window])
+            window_deltas_gts.append(gt.poses[i].inverse() * gt.poses[i + window])
+
+    elif kind == WindowKind.distance:
+        # Compute deltas for all of ground truth poses
+        dist = np.zeros(len(gt))
+        for i in range(1, len(gt)):
+            diff: NDArray[np.float64] = gt.poses[i].trans - gt.poses[i - 1].trans  # type: ignore
+            dist[i] = np.sqrt(diff @ diff)
+
+        cum_dist = np.cumsum(dist)
+        end_idx = 1
+        end_idx_prev = 0
+
+        # Find our pairs for computation
+        for i in range(len(gt)):
+            while end_idx < len(gt) and cum_dist[end_idx] - cum_dist[i] < window:
+                end_idx += 1
+
+            if end_idx >= len(gt):
+                break
+            elif end_idx == end_idx_prev:
+                continue
+
+            window_deltas_poses.append(traj.poses[i].inverse() * traj.poses[end_idx])
+            window_deltas_gts.append(gt.poses[i].inverse() * gt.poses[end_idx])
+
+            end_idx_prev = end_idx
 
     # Compute the RTE
     return _compute_metric(window_deltas_gts, window_deltas_poses)
diff --git a/tests/test_dataset_loading.py b/tests/test_dataset_loading.py
index 91ab20d1..cc2d5a68 100644
--- a/tests/test_dataset_loading.py
+++ b/tests/test_dataset_loading.py
@@ -6,7 +6,14 @@
 import pytest
 from evalio.cli.parser import DatasetBuilder
 from evalio.datasets.base import Dataset
-from evalio.types import SE3, ImuMeasurement, LidarMeasurement, Stamp, Trajectory
+from evalio.types import (
+    SE3,
+    GroundTruth,
+    ImuMeasurement,
+    LidarMeasurement,
+    Stamp,
+    Trajectory,
+)
 from utils import check_lidar_eq, isclose_se3, rand_se3
 
 # ------------------------- Loading imu & lidar ------------------------- #
@@ -72,7 +79,7 @@ def fake_groundtruth() -> Trajectory:
         for i in range(1_000, 10_000, 1_000)
     ]
     poses = [rand_se3() for _ in range(len(stamps))]
-    return Trajectory(metadata={}, stamps=stamps, poses=poses)
+    return Trajectory(stamps=stamps, poses=poses, metadata=GroundTruth(sequence="fake"))
 
 
 def serialize_gt(gt: Trajectory, style: StampStyle) -> list[str]:
diff --git a/tests/test_experiment.py b/tests/test_experiment.py
new file mode 100644
index 00000000..3009636e
--- /dev/null
+++ b/tests/test_experiment.py
@@ -0,0 +1,92 @@
+from evalio.types import Experiment, ExperimentStatus, Param
+from evalio.pipelines import Pipeline, register_pipeline
+
+import pytest
+
+
+class FakePipeline(Pipeline):
+    @staticmethod
+    def name() -> str:
+        return "fake"
+
+    @staticmethod
+    def version() -> str:
+        return "0.1.0"
+
+    @staticmethod
+    def default_params() -> dict[str, Param]:
+        return {"param1": 1, "param2": "value"}
+
+
+def test_serde():
+    exp = Experiment(
+        name="test",
+        status=ExperimentStatus.Complete,
+        sequence="newer_college_2020/short_experiment",
+        sequence_length=1000,
+        pipeline="fake",
+        pipeline_version="0.1.0",
+        pipeline_params={"param1": 1, "param2": "value"},
+        total_elapsed=10.5,
+        max_step_elapsed=0.24,
+    )
+    out = Experiment.from_yaml(exp.to_yaml())
+    assert exp == out
+
+
+def test_verify(capsys: pytest.CaptureFixture[str]):
+    register_pipeline(FakePipeline)
+    misc = {
+        "name": "test",
+        "status": ExperimentStatus.Complete,
+        "sequence": "newer_college_2020/short_experiment",
+        "pipeline_version": "0.1.0",
+        "do_verify": True,
+    }
+
+    # Bad pipeline name
+    with pytest.raises(ValueError) as exc:
+        Experiment(
+            **misc,  # type: ignore
+            sequence_length=1000,
+            pipeline="bad_name",
+            pipeline_params={"param1": 2, "param2": "value"},  # wrong param1
+        )
+    assert str(exc.value) == "Experiment 'test' has unknown pipeline 'bad_name'"
+
+    # Bad param name and type
+    with pytest.raises(ValueError) as exc:
+        Experiment(
+            **misc,  # type: ignore
+            sequence_length=1000,
+            pipeline="fake",
+            pipeline_params={"bad_param": 2, "param2": "value"},  # wrong param1
+        )
+    assert str(exc.value) == "Invalid parameter 'bad_param' for pipeline 'fake'"
+
+    # Bad param type
+    with pytest.raises(ValueError) as exc:
+        Experiment(
+            **misc,  # type: ignore
+            sequence_length=1000,
+            pipeline="fake",
+            pipeline_params={"param1": 2.0, "param2": "value"},  # wrong param1
+        )
+    assert (
+        str(exc.value)
+        == "Invalid type for parameter 'param1' for pipeline 'fake': expected 'int', got 'float'"
+    )
+
+    # Too long length
+    Experiment(
+        **misc,  # type: ignore
+        sequence_length=2000000,
+        pipeline="fake",
+        pipeline_params={"param1": 1, "param2": "value"},
+    )
+
+    captured = capsys.readouterr()
+    assert (
+        captured.out
+        == "Warning: Experiment 'test' has sequence_length 2000000 > dataset length 15302, reducing to 15302\n"
+    )
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 1cd8bb13..346b5b4f 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -14,7 +14,6 @@ def s(t: float) -> Stamp:
 
 def test_already_aligned():
     traj = Trajectory(
-        metadata={},
         stamps=[s(0), s(1), s(2)],
         poses=[ID, ID, ID],
     )
@@ -28,13 +27,11 @@ def test_already_aligned():
 
 def test_subsample_first():
     traj1 = Trajectory(
-        metadata={},
         stamps=[s(i) for i in range(10)],
         poses=[ID for _ in range(10)],
     )
 
     traj2 = Trajectory(
-        metadata={},
         stamps=[s(i) for i in range(0, 10, 2)],
         poses=[ID for _ in range(0, 10, 2)],
     )
@@ -56,13 +53,11 @@ def test_subsample_first():
 def test_overstep():
     r = list(range(1, 11))
     traj1 = Trajectory(
-        metadata={},
         stamps=[s(i - 0.1) for i in r],
         poses=[ID for _ in r],
     )
 
     traj2 = Trajectory(
-        metadata={},
         stamps=[s(i) for i in r],
         poses=[ID for _ in r],
     )
@@ -84,7 +79,6 @@ def test_overstep():
 def testalign_poses():
     np.random.seed(0)
     gt = Trajectory(
-        metadata={},
         stamps=[s(i) for i in range(10)],
         poses=[rand_se3() for _ in range(10)],
     )
@@ -92,7 +86,6 @@ def testalign_poses():
     offset = rand_se3()
 
     traj2 = Trajectory(
-        metadata={},
         stamps=[s(i) for i in range(10)],
         poses=[offset * pose for pose in gt.poses],
     )

From 7073739721dc6adb97529e163cf3a2d5f204fa5f Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Mon, 29 Sep 2025 14:13:21 -0400
Subject: [PATCH 04/40] Major changes to underlying types and serialization
 code

---
 pyproject.toml                      |   2 +
 python/evalio/datasets/__init__.py  |   8 +
 python/evalio/datasets/base.py      |  69 ++----
 python/evalio/datasets/parser.py    |  82 ++++++-
 python/evalio/pipelines/__init__.py |  10 +-
 python/evalio/pipelines/parser.py   |  88 +++++++-
 python/evalio/types.py              | 329 ----------------------------
 python/evalio/types/__init__.py     |  36 +++
 python/evalio/types/base.py         | 305 ++++++++++++++++++++++++++
 python/evalio/types/extended.py     | 123 +++++++++++
 python/evalio/utils.py              |  42 ++++
 uv.lock                             |   6 +-
 12 files changed, 709 insertions(+), 391 deletions(-)
 delete mode 100644 python/evalio/types.py
 create mode 100644 python/evalio/types/__init__.py
 create mode 100644 python/evalio/types/base.py
 create mode 100644 python/evalio/types/extended.py

diff --git a/pyproject.toml b/pyproject.toml
index 8c799b10..d3b1b625 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -85,6 +85,7 @@ dev-dependencies = [
 
 [tool.ruff]
 exclude = ["cpp/**/*"]
+ignore = ["E731"]
 
 [tool.ruff.format]
 # https://docs.astral.sh/ruff/configuration/
@@ -100,6 +101,7 @@ include = ["python", "tests"]
 typeCheckingMode = "strict"
 stubPath = "python/typings"
 reportPrivateUsage = "none"
+reportConstantRedefinition = "none"
 
 [tool.bumpversion]
 allow_dirty = false
diff --git a/python/evalio/datasets/__init__.py b/python/evalio/datasets/__init__.py
index 4455f1af..81c14aa9 100644
--- a/python/evalio/datasets/__init__.py
+++ b/python/evalio/datasets/__init__.py
@@ -16,6 +16,10 @@
     all_sequences,
     get_sequence,
     register_dataset,
+    parse_config,
+    DatasetNotFound,
+    SequenceNotFound,
+    InvalidDatasetConfig,
 )
 
 __all__ = [
@@ -39,4 +43,8 @@
     "all_sequences",
     "get_sequence",
     "register_dataset",
+    "parse_config",
+    "DatasetNotFound",
+    "SequenceNotFound",
+    "InvalidDatasetConfig",
 ]
diff --git a/python/evalio/datasets/base.py b/python/evalio/datasets/base.py
index 467cc26e..9190feb6 100644
--- a/python/evalio/datasets/base.py
+++ b/python/evalio/datasets/base.py
@@ -1,24 +1,25 @@
 import os
-from enum import Enum, StrEnum, auto
+from enum import StrEnum
 from itertools import islice
 from pathlib import Path
 from typing import Iterable, Iterator, Optional, Sequence, Union
 
-from evalio.types import (
+from evalio._cpp.types import (  # type: ignore
     SE3,
-    GroundTruth,
     ImuMeasurement,
     ImuParams,
     LidarMeasurement,
     LidarParams,
-    Trajectory,
 )
-from evalio.utils import print_warning
+
+from evalio.types import GroundTruth, Trajectory
+
+from evalio.utils import print_warning, pascal_to_snake
 
 Measurement = Union[ImuMeasurement, LidarMeasurement]
 
-_data_dir = Path(os.environ.get("EVALIO_DATA", "evalio_data"))
-_warned = False
+_DATA_DIR = Path(os.environ.get("EVALIO_DATA", "evalio_data"))
+_WARNED = False
 
 
 class DatasetIterator(Iterable[Measurement]):
@@ -236,12 +237,12 @@ def _fail_not_downloaded(self):
 
     @classmethod
     def _warn_default_dir(cls):
-        global _data_dir, _warned
-        if not _warned and _data_dir == Path("./evalio_data"):
+        global _DATA_DIR, _WARNED
+        if not _WARNED and _DATA_DIR == Path("./evalio_data"):
             print_warning(
-                "Using default './evalio_data' for base data directory. Override by setting [magenta]EVALIO_DATA[/magenta], [magenta]evalio.set_data_dir(path)[/magenta] in python, or [magenta]-D[/magenta] in the CLI."
+                "Using default './evalio_data' for base data directory. Override by setting [magenta]EVALIO_DATA[/magenta], [magenta]evalio.set_DATA_DIR(path)[/magenta] in python, or [magenta]-D[/magenta] in the CLI."
             )
-            _warned = True
+            _WARNED = True
 
     # ------------------------- Helpers that leverage from the iterator ------------------------- #
 
@@ -352,8 +353,8 @@ def folder(self) -> Path:
         Returns:
             Path: Path to the dataset folder.
         """
-        global _data_dir
-        return _data_dir / self.full_name
+        global _DATA_DIR
+        return _DATA_DIR / self.full_name
 
     def size_on_disk(self) -> Optional[float]:
         """Shows the size of the dataset on disk, in GB.
@@ -368,38 +369,6 @@ def size_on_disk(self) -> Optional[float]:
             return sum(f.stat().st_size for f in self.folder.glob("**/*")) / 1e9
 
 
-# For converting dataset names to snake case
-class CharKinds(Enum):
-    LOWER = auto()
-    UPPER = auto()
-    DIGIT = auto()
-    OTHER = auto()
-
-    @staticmethod
-    def from_char(char: str):
-        if char.islower():
-            return CharKinds.LOWER
-        if char.isupper():
-            return CharKinds.UPPER
-        if char.isdigit():
-            return CharKinds.DIGIT
-        return CharKinds.OTHER
-
-
-def pascal_to_snake(identifier: str) -> str:
-    # Only split when going from lower to something else
-    splits: list[int] = []
-    last_kind = CharKinds.from_char(identifier[0])
-    for i, char in enumerate(identifier[1:], start=1):
-        kind = CharKinds.from_char(char)
-        if last_kind == CharKinds.LOWER and kind != CharKinds.LOWER:
-            splits.append(i)
-        last_kind = kind
-
-    parts = [identifier[i:j] for i, j in zip([0] + splits, splits + [None])]
-    return "_".join(parts).lower()
-
-
 # ------------------------- Helpers ------------------------- #
 def set_data_dir(directory: Path):
     """Set the global location where datasets are stored. This will be used to store the downloaded data.
@@ -407,9 +376,9 @@ def set_data_dir(directory: Path):
     Args:
         directory (Path): Directory
     """
-    global _data_dir, _warned
-    _data_dir = directory
-    _warned = True
+    global _DATA_DIR, _WARNED
+    _DATA_DIR = directory
+    _WARNED = True
 
 
 def get_data_dir() -> Path:
@@ -418,5 +387,5 @@ def get_data_dir() -> Path:
     Returns:
         Path: Directory where datasets are stored.
     """
-    global _data_dir
-    return _data_dir
+    global _DATA_DIR
+    return _DATA_DIR
diff --git a/python/evalio/datasets/parser.py b/python/evalio/datasets/parser.py
index 0519fc86..e4472173 100644
--- a/python/evalio/datasets/parser.py
+++ b/python/evalio/datasets/parser.py
@@ -1,7 +1,8 @@
 import importlib
 from inspect import isclass
+import itertools
 from types import ModuleType
-from typing import Optional
+from typing import Callable, Optional, Sequence, TypedDict, cast
 
 from evalio import datasets
 from evalio.datasets.base import Dataset
@@ -9,6 +10,24 @@
 _DATASETS: set[type[Dataset]] = set()
 
 
+class DatasetNotFound(Exception):
+    def __init__(self, name: str):
+        super().__init__(f"Dataset '{name}' not found")
+        self.name = name
+
+
+class SequenceNotFound(Exception):
+    def __init__(self, name: str):
+        super().__init__(f"Sequence '{name}' not found")
+        self.name = name
+
+
+class InvalidDatasetConfig(Exception):
+    def __init__(self, config: str):
+        super().__init__(f"Invalid config: '{config}'")
+        self.config = config
+
+
 # ------------------------- Handle Registration of Datasets ------------------------- #
 def _is_dataset(obj: object) -> bool:
     return (
@@ -52,8 +71,8 @@ def all_datasets() -> dict[str, type[Dataset]]:
     return {d.dataset_name(): d for d in _DATASETS}
 
 
-def get_dataset(name: str) -> Optional[type[Dataset]]:
-    return all_datasets().get(name, None)
+def get_dataset(name: str) -> type[Dataset] | DatasetNotFound:
+    return all_datasets().get(name, DatasetNotFound(name))
 
 
 def all_sequences() -> dict[str, Dataset]:
@@ -62,10 +81,63 @@ def all_sequences() -> dict[str, Dataset]:
     }
 
 
-def get_sequence(name: str) -> Optional[Dataset]:
-    return all_sequences().get(name, None)
+def get_sequence(name: str) -> Dataset | SequenceNotFound:
+    return all_sequences().get(name, SequenceNotFound(name))
 
 
 register_dataset(module=datasets)
 
+
 # ------------------------- Handle yaml parsing ------------------------- #
+class DatasetConfig(TypedDict):
+    name: str
+    length: Optional[int]
+
+
+ConfigError = DatasetNotFound | SequenceNotFound | InvalidDatasetConfig
+
+
+def parse_config(
+    d: str | DatasetConfig | Sequence[str | DatasetConfig],
+) -> list[tuple[Dataset, int]] | ConfigError:
+    name: Optional[str] = None
+    length: Optional[int] = None
+    # If given a list of values
+    if isinstance(d, list):
+        results = [parse_config(x) for x in d]
+        for r in results:
+            if isinstance(r, ConfigError):
+                return r
+        results = cast(list[list[tuple[Dataset, int]]], results)
+        return list(itertools.chain.from_iterable(results))
+
+    # If it's a single config
+    elif isinstance(d, str):
+        name = d
+        length = None
+    elif isinstance(d, dict):
+        name = d.get("name", None)
+        length = d.get("length", None)
+    else:
+        return InvalidDatasetConfig(str(d))
+
+    if name is None:  # type: ignore
+        return InvalidDatasetConfig(str(d))
+
+    length_lambda: Callable[[Dataset], int]
+    if length is None:
+        length_lambda = lambda s: len(s)
+    else:
+        length_lambda = lambda s: length
+
+    if name[-2:] == "/*":
+        ds_name, _ = name.split("/")
+        ds = get_dataset(ds_name)
+        if isinstance(ds, DatasetNotFound):
+            return ds
+        return [(s, length_lambda(s)) for s in ds.sequences()]
+
+    ds = get_sequence(name)
+    if isinstance(ds, SequenceNotFound):
+        return ds
+    return [(ds, length_lambda(ds))]
diff --git a/python/evalio/pipelines/__init__.py b/python/evalio/pipelines/__init__.py
index 145713e5..d5b72462 100644
--- a/python/evalio/pipelines/__init__.py
+++ b/python/evalio/pipelines/__init__.py
@@ -1,10 +1,18 @@
 from evalio._cpp.pipelines import *  # type: ignore  # noqa: F403
 
-from .parser import register_pipeline, get_pipeline, all_pipelines
+from .parser import (
+    register_pipeline,
+    get_pipeline,
+    all_pipelines,
+    PipelineNotFound,
+    InvalidPipelineConfig,
+)
 
 
 __all__ = [
     "all_pipelines",
     "get_pipeline",
     "register_pipeline",
+    "PipelineNotFound",
+    "InvalidPipelineConfig",
 ]
diff --git a/python/evalio/pipelines/parser.py b/python/evalio/pipelines/parser.py
index 67929442..a62ea286 100644
--- a/python/evalio/pipelines/parser.py
+++ b/python/evalio/pipelines/parser.py
@@ -1,14 +1,31 @@
+from __future__ import annotations
+
 import importlib
 from inspect import isclass
+import itertools
 from types import ModuleType
-from typing import Any, Optional
+from typing import Any, Optional, cast, Sequence
 
 from evalio import pipelines
 from evalio.pipelines import Pipeline
+from evalio.types import Param
+
 
 _PIPELINES: set[type[Pipeline]] = set()
 
 
+class PipelineNotFound(Exception):
+    def __init__(self, name: str):
+        super().__init__(f"Pipeline '{name}' not found")
+        self.name = name
+
+
+class InvalidPipelineConfig(Exception):
+    def __init__(self, config: str):
+        super().__init__(f"Invalid config: '{config}'")
+        self.config = config
+
+
 # ------------------------- Handle Registration of Pipelines ------------------------- #
 def _is_pipe(obj: Any) -> bool:
     return (
@@ -66,7 +83,7 @@ def all_pipelines() -> dict[str, type[Pipeline]]:
     return {p.name(): p for p in _PIPELINES}
 
 
-def get_pipeline(name: str) -> Optional[type[Pipeline]]:
+def get_pipeline(name: str) -> type[Pipeline] | PipelineNotFound:
     """Get a pipeline class by its name.
 
     Args:
@@ -75,9 +92,74 @@ def get_pipeline(name: str) -> Optional[type[Pipeline]]:
     Returns:
         Optional[type[Pipeline]]: The pipeline class, or None if not found.
     """
-    return all_pipelines().get(name, None)
+    return all_pipelines().get(name, PipelineNotFound(name))
 
 
 register_pipeline(module=pipelines)
 
+
 # ------------------------- Handle yaml parsing ------------------------- #
+def _sweep(
+    sweep: dict[str, Param],
+    params: dict[str, Param],
+    pipe: type[Pipeline],
+) -> list[tuple[type[Pipeline], dict[str, Param]]]:
+    keys, values = zip(*sweep.items())
+    results: list[tuple[type[Pipeline], dict[str, Param]]] = []
+    for options in itertools.product(*values):
+        p = params.copy()
+        for k, o in zip(keys, options):
+            p[k] = o
+        results.append((pipe, p))
+    return results
+
+
+ConfigError = PipelineNotFound | InvalidPipelineConfig
+
+
+def parse_config(
+    p: str | dict[str, Param] | Sequence[str | dict[str, Param]],
+) -> list[tuple[type[Pipeline], dict[str, Param]]] | ConfigError:
+    """Parse a pipeline configuration.
+
+    Args:
+        p (str | dict[str, Param] | Sequence[str | dict[str, Param]]): The pipeline configuration.
+
+    Raises:
+        ValueError: If the pipeline is not found.
+        ValueError: If the configuration is invalid.
+
+    Returns:
+        list[tuple[type[Pipeline], dict[str, Param]]]: A list of tuples containing the pipeline class and its parameters.
+    """
+    if isinstance(p, str):
+        pipe = get_pipeline(p)
+        if isinstance(pipe, PipelineNotFound):
+            return pipe
+        return [(pipe, {})]
+
+    elif isinstance(p, dict):
+        name = p.pop("name", None)
+        if name is None:
+            return InvalidPipelineConfig(f"Need pipeline name: {str(p)}")
+
+        pipe = get_pipeline(cast(str, name))
+        if isinstance(pipe, PipelineNotFound):
+            return pipe
+
+        if "sweep" in p:
+            sweep = cast(dict[str, Param], p.pop("sweep"))
+            return _sweep(sweep, p, pipe)
+        else:
+            return [(pipe, p)]
+
+    elif isinstance(p, list):
+        results = [parse_config(x) for x in p]
+        for r in results:
+            if isinstance(r, ConfigError):
+                return r
+        results = cast(list[list[tuple[type[Pipeline], dict[str, Param]]]], results)
+        return list(itertools.chain.from_iterable(results))
+
+    else:
+        return InvalidPipelineConfig(f"Invalid pipeline configuration {p}")
diff --git a/python/evalio/types.py b/python/evalio/types.py
deleted file mode 100644
index b6458a59..00000000
--- a/python/evalio/types.py
+++ /dev/null
@@ -1,329 +0,0 @@
-import csv
-from dataclasses import dataclass, field
-from enum import Enum
-from pathlib import Path
-from typing import Optional
-
-from evalio.utils import print_warning
-import numpy as np
-import yaml
-
-from ._cpp.types import (  # type: ignore
-    SE3,
-    SO3,
-    Duration,
-    ImuMeasurement,
-    ImuParams,
-    LidarMeasurement,
-    LidarParams,
-    Point,
-    Stamp,
-)
-
-Param = bool | int | float | str
-
-
-@dataclass(kw_only=True)
-class GroundTruth:
-    sequence: str
-    """Dataset used to run the experiment."""
-
-    @staticmethod
-    def from_yaml(yaml_str: str) -> Optional["GroundTruth"]:
-        """Create a GroundTruth object from a YAML string.
-
-        Args:
-            yaml_str (str): YAML string to parse.
-        """
-        data = yaml.safe_load(yaml_str)
-
-        gt: Optional[bool] = data.pop("gt", None)
-        sequence = data.pop("sequence", None)
-
-        if gt is None or not gt or sequence is None:
-            return None
-
-        return GroundTruth(sequence=sequence)
-
-    def to_yaml(self) -> str:
-        """Convert the GroundTruth object to a YAML string.
-
-        Returns:
-            str: YAML string representation of the GroundTruth object.
-        """
-        data = {
-            "gt": True,
-            "sequence": self.sequence,
-        }
-
-        return yaml.dump(data)
-
-
-class ExperimentStatus(Enum):
-    Complete = "complete"
-    Fail = "fail"
-    Started = "started"
-    NeverRan = "never_ran"
-
-
-@dataclass(kw_only=True)
-class Experiment:
-    name: str
-    """Name of the experiment."""
-    status: ExperimentStatus
-    """Status of the experiment, e.g. "success", "failure", etc."""
-    sequence: str
-    """Dataset used to run the experiment."""
-    sequence_length: int
-    """Length of the sequence, if set"""
-    pipeline: str
-    """Pipeline used to generate the trajectory."""
-    pipeline_version: str
-    """Version of the pipeline used."""
-    pipeline_params: dict[str, Param] = field(default_factory=dict)
-    """Parameters used for the pipeline."""
-    total_elapsed: Optional[float] = None
-    """Total time taken for the experiment, as a string."""
-    max_step_elapsed: Optional[float] = None
-    """Maximum time taken for a single step in the experiment, as a string."""
-
-    @staticmethod
-    def from_yaml(yaml_str: str) -> Optional["Experiment"]:
-        """Create an Experiment object from a YAML string.
-
-        Args:
-            yaml_str (str): YAML string to parse.
-        """
-        data = yaml.safe_load(yaml_str)
-
-        name = data.pop("name", None)
-        pipeline = data.pop("pipeline", None)
-        pipeline_version = data.pop("pipeline_version", None)
-        pipeline_params = data.pop("pipeline_params", None)
-        sequence = data.pop("sequence", None)
-        sequence_length = data.pop("sequence_length", None)
-
-        if (
-            name is None
-            or sequence is None
-            or sequence_length is None
-            or pipeline is None
-            or pipeline_version is None
-            or pipeline_params is None
-        ):
-            return None
-
-        total_elapsed = data.pop("total_elapsed", None)
-        max_step_elapsed = data.pop("max_step_elapsed", None)
-
-        if "status" in data:
-            status = ExperimentStatus(data.pop("status"))
-        else:
-            status = ExperimentStatus.Started
-
-        if len(data) > 0:
-            # Unknown fields
-            print_warning(
-                f"Experiment.from_yaml: Unknown fields in YAML: {list(data.keys())}"
-            )
-
-        return Experiment(
-            name=name,
-            sequence=sequence,
-            sequence_length=sequence_length,
-            pipeline=pipeline,
-            pipeline_version=pipeline_version,
-            pipeline_params=pipeline_params,
-            status=status,
-            total_elapsed=total_elapsed,
-            max_step_elapsed=max_step_elapsed,
-        )
-
-    def to_yaml(self) -> str:
-        """Convert the Experiment object to a YAML string.
-
-        Returns:
-            str: YAML string representation of the Experiment object.
-        """
-        data: dict[str, Param | dict[str, Param]] = {
-            "name": self.name,
-            "sequence": self.sequence,
-            "sequence_length": self.sequence_length,
-            "pipeline": self.pipeline,
-            "pipeline_params": self.pipeline_params,
-        }
-        if self.status in [ExperimentStatus.Complete, ExperimentStatus.Fail]:
-            data["status"] = self.status.value
-        if self.total_elapsed is not None:
-            data["total_elapsed"] = self.total_elapsed
-        if self.max_step_elapsed is not None:
-            data["max_step_elapsed"] = self.max_step_elapsed
-
-        return yaml.dump(data)
-
-
-def _parse_metadata(yaml_str: str) -> Optional[GroundTruth | Experiment]:
-    if "gt:" in yaml_str:
-        return GroundTruth.from_yaml(yaml_str)
-    elif "name:" in yaml_str:
-        return Experiment.from_yaml(yaml_str)
-    else:
-        return None
-
-
-@dataclass(kw_only=True)
-class Trajectory:
-    stamps: list[Stamp]
-    """List of timestamps for each pose."""
-    poses: list[SE3]
-    """List of poses, in the same order as the timestamps."""
-    metadata: Optional[GroundTruth | Experiment] = None
-    """Metadata associated with the trajectory, such as the dataset name or other information."""
-
-    def __post_init__(self):
-        if len(self.stamps) != len(self.poses):
-            raise ValueError("Stamps and poses must have the same length.")
-
-    def __getitem__(self, idx: int) -> tuple[Stamp, SE3]:
-        return self.stamps[idx], self.poses[idx]
-
-    def __len__(self) -> int:
-        return len(self.stamps)
-
-    def __iter__(self):
-        return iter(zip(self.stamps, self.poses))
-
-    def append(self, stamp: Stamp, pose: SE3):
-        self.stamps.append(stamp)
-        self.poses.append(pose)
-
-    def transform_in_place(self, T: SE3):
-        for i in range(len(self.poses)):
-            self.poses[i] = self.poses[i] * T
-
-    @staticmethod
-    def from_csv(
-        path: Path,
-        fieldnames: list[str],
-        delimiter: str = ",",
-        skip_lines: Optional[int] = None,
-    ) -> "Trajectory":
-        """Flexible loader for stamped poses stored in csv files.
-
-        Will automatically skip any lines that start with a #. Is most useful for loading ground truth data.
-
-        ``` py
-        from evalio.types import Trajectory
-
-        fieldnames = ["sec", "nsec", "x", "y", "z", "qx", "qy", "qz", "qw"]
-        trajectory = Trajectory.from_csv(path, fieldnames)
-        ```
-
-        Args:
-            path (Path): Location of file.
-            fieldnames (list[str]): List of field names to use, in their expected order. See above for an example.
-            delimiter (str, optional): Delimiter between elements. Defaults to ",".
-            skip_lines (int, optional): Number of lines to skip, useful for skipping headers. Defaults to 0.
-
-        Returns:
-            Trajectory: Stored dataset
-        """
-        poses: list[SE3] = []
-        stamps: list[Stamp] = []
-
-        with open(path) as f:
-            csvfile = list(filter(lambda row: row[0] != "#", f))
-            if skip_lines is not None:
-                csvfile = csvfile[skip_lines:]
-            reader = csv.DictReader(csvfile, fieldnames=fieldnames, delimiter=delimiter)
-            for line in reader:
-                r = SO3(
-                    qw=float(line["qw"]),
-                    qx=float(line["qx"]),
-                    qy=float(line["qy"]),
-                    qz=float(line["qz"]),
-                )
-                t = np.array([float(line["x"]), float(line["y"]), float(line["z"])])
-                pose = SE3(r, t)
-
-                if "t" in fieldnames:
-                    line["sec"] = line["t"]
-
-                if "nsec" not in fieldnames:
-                    s, ns = line["sec"].split(
-                        "."
-                    )  # parse separately to get exact stamp
-                    ns = ns.ljust(9, "0")  # pad to 9 digits for nanoseconds
-                    stamp = Stamp(sec=int(s), nsec=int(ns))
-                elif "sec" not in fieldnames:
-                    stamp = Stamp.from_nsec(int(line["nsec"]))
-                else:
-                    stamp = Stamp(sec=int(line["sec"]), nsec=int(line["nsec"]))
-                poses.append(pose)
-                stamps.append(stamp)
-
-        return Trajectory(stamps=stamps, poses=poses)
-
-    @staticmethod
-    def from_tum(path: Path) -> "Trajectory":
-        """Load a TUM dataset pose file. Simple wrapper around [from_csv][evalio.types.Trajectory].
-
-        Args:
-            path (Path): Location of file.
-
-        Returns:
-            Trajectory: Stored trajectory
-        """
-        return Trajectory.from_csv(path, ["sec", "x", "y", "z", "qx", "qy", "qz", "qw"])
-
-    @staticmethod
-    def from_experiment(path: Path) -> Optional["Trajectory"]:
-        """Load a saved experiment trajectory from file.
-
-        Works identically to [from_tum][evalio.types.Trajectory.from_tum], but also loads metadata from the file.
-
-        Args:
-            path (Path): Location of trajectory results.
-
-        Returns:
-            Trajectory: Loaded trajectory with metadata, stamps, and poses.
-        """
-        with open(path) as file:
-            metadata_filter = filter(
-                lambda row: row[0] == "#" and not row.startswith("# timestamp,"), file
-            )
-            metadata_list = [row[1:].strip() for row in metadata_filter]
-            if len(metadata_list) == 0:
-                return None
-
-            metadata_str = "\n".join(metadata_list)
-            metadata = _parse_metadata(metadata_str)
-
-        trajectory = Trajectory.from_csv(
-            path,
-            fieldnames=["sec", "x", "y", "z", "qx", "qy", "qz", "qw"],
-        )
-        trajectory.metadata = metadata
-
-        return trajectory
-
-
-# TODO: Some sort of incremental writer for experiments
-# TODO: Some sort of batch writer for ground truth (can be the same as above)
-
-
-__all__ = [
-    "Experiment",
-    "ExperimentStatus",
-    "ImuMeasurement",
-    "ImuParams",
-    "LidarMeasurement",
-    "LidarParams",
-    "Duration",
-    "Param",
-    "Point",
-    "SO3",
-    "SE3",
-    "Stamp",
-    "Trajectory",
-]
diff --git a/python/evalio/types/__init__.py b/python/evalio/types/__init__.py
new file mode 100644
index 00000000..9389a0cd
--- /dev/null
+++ b/python/evalio/types/__init__.py
@@ -0,0 +1,36 @@
+from evalio._cpp.types import (  # type: ignore
+    SE3,
+    SO3,
+    Duration,
+    ImuMeasurement,
+    ImuParams,
+    LidarMeasurement,
+    LidarParams,
+    Point,
+    Stamp,
+)
+
+from .base import Param, Trajectory, Metadata, GroundTruth
+from .extended import Experiment, ExperimentStatus
+
+
+__all__ = [
+    # cpp includes
+    "ImuMeasurement",
+    "ImuParams",
+    "LidarMeasurement",
+    "LidarParams",
+    "Duration",
+    "Point",
+    "SO3",
+    "SE3",
+    "Stamp",
+    # base includes
+    "GroundTruth",
+    "Metadata",
+    "Param",
+    "Trajectory",
+    # extended includes
+    "Experiment",
+    "ExperimentStatus",
+]
diff --git a/python/evalio/types/base.py b/python/evalio/types/base.py
new file mode 100644
index 00000000..2a00059f
--- /dev/null
+++ b/python/evalio/types/base.py
@@ -0,0 +1,305 @@
+"""
+These are the base python-based types used throughout evalio.
+
+They MUST not depend on anything else in evalio, or else circular imports will occur.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+import csv
+from _csv import Writer
+from enum import Enum
+from io import TextIOWrapper
+from evalio.utils import print_warning
+import numpy as np
+import yaml
+
+from pathlib import Path
+from typing import Any, ClassVar, Optional, Self
+
+from evalio._cpp.types import (  # type: ignore
+    SE3,
+    SO3,
+    Stamp,
+)
+
+from evalio.utils import pascal_to_snake
+
+Param = bool | int | float | str
+
+
+class ExperimentStatus(Enum):
+    Complete = "complete"
+    Fail = "fail"
+    Started = "started"
+
+
+class FailedMetadataParse(Exception):
+    def __init__(self, reason: str):
+        super().__init__(f"Failed to parse metadata: {reason}")
+        self.reason = reason
+
+
+@dataclass(kw_only=True)
+class Metadata:
+    file: Optional[Path] = None
+    """File where the metadata was loaded to and from, if any."""
+    _registry: ClassVar[dict[str, type[Self]]] = {}
+
+    def __init_subclass__(cls) -> None:
+        cls._registry[cls.tag()] = cls
+
+    @classmethod
+    def tag(cls) -> str:
+        return pascal_to_snake(cls.__name__)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> Self:
+        if "type" in data:
+            del data["type"]
+        return cls(**data)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    def to_yaml(self) -> str:
+        data = self.to_dict()
+        data["type"] = self.tag()
+        return yaml.safe_dump(data)
+
+    @classmethod
+    def parse(cls, yaml_str: str) -> Metadata | FailedMetadataParse:
+        data = yaml.safe_load(yaml_str)
+
+        if "type: " not in data:
+            return FailedMetadataParse("No type field found in metadata.")
+
+        for name, subclass in cls._registry.items():
+            if data["type"] == name:
+                try:
+                    return subclass.from_dict(data)
+                except Exception as e:
+                    return FailedMetadataParse(f"Failed to parse {name}: {e}")
+
+        return FailedMetadataParse(f"Unknown metadata type '{data['type']}'")
+
+
+@dataclass(kw_only=True)
+class GroundTruth(Metadata):
+    sequence: str
+    """Dataset used to run the experiment."""
+
+
+@dataclass(kw_only=True)
+class Trajectory:
+    stamps: list[Stamp]
+    """List of timestamps for each pose."""
+    poses: list[SE3]
+    """List of poses, in the same order as the timestamps."""
+    metadata: Optional[Metadata] = None
+    """Metadata associated with the trajectory, such as the dataset name or other information."""
+    _file: Optional[TextIOWrapper] = None
+    _csv_writer: Optional[Writer] = None
+
+    def __post_init__(self):
+        if len(self.stamps) != len(self.poses):
+            raise ValueError("Stamps and poses must have the same length.")
+
+    def __getitem__(self, idx: int) -> tuple[Stamp, SE3]:
+        return self.stamps[idx], self.poses[idx]
+
+    def __len__(self) -> int:
+        return len(self.stamps)
+
+    def __iter__(self):
+        return iter(zip(self.stamps, self.poses))
+
+    def append(self, stamp: Stamp, pose: SE3):
+        self.stamps.append(stamp)
+        self.poses.append(pose)
+
+        if self._csv_writer is not None:
+            self._csv_writer.writerow(self._serialize_pose(stamp, pose))
+
+    def transform_in_place(self, T: SE3):
+        for i in range(len(self.poses)):
+            self.poses[i] = self.poses[i] * T
+
+    # ------------------------- Loading from file ------------------------- #
+    @staticmethod
+    def from_csv(
+        path: Path,
+        fieldnames: list[str],
+        delimiter: str = ",",
+        skip_lines: Optional[int] = None,
+    ) -> Trajectory:
+        """Flexible loader for stamped poses stored in csv files.
+
+        Will automatically skip any lines that start with a #.
+
+        ``` py
+        from evalio.types import Trajectory
+
+        fieldnames = ["sec", "nsec", "x", "y", "z", "qx", "qy", "qz", "qw"]
+        trajectory = Trajectory.from_csv(path, fieldnames)
+        ```
+
+        Args:
+            path (Path): Location of file.
+            fieldnames (list[str]): List of field names to use, in their expected order. See above for an example.
+            delimiter (str, optional): Delimiter between elements. Defaults to ",".
+            skip_lines (int, optional): Number of lines to skip, useful for skipping headers. Defaults to 0.
+
+        Returns:
+            Trajectory: Stored dataset
+        """
+        poses: list[SE3] = []
+        stamps: list[Stamp] = []
+
+        with open(path) as f:
+            csvfile = list(filter(lambda row: row[0] != "#", f))
+            if skip_lines is not None:
+                csvfile = csvfile[skip_lines:]
+            reader = csv.DictReader(csvfile, fieldnames=fieldnames, delimiter=delimiter)
+            for line in reader:
+                r = SO3(
+                    qw=float(line["qw"]),
+                    qx=float(line["qx"]),
+                    qy=float(line["qy"]),
+                    qz=float(line["qz"]),
+                )
+                t = np.array([float(line["x"]), float(line["y"]), float(line["z"])])
+                pose = SE3(r, t)
+
+                if "t" in fieldnames:
+                    line["sec"] = line["t"]
+
+                if "nsec" not in fieldnames:
+                    s, ns = line["sec"].split(
+                        "."
+                    )  # parse separately to get exact stamp
+                    ns = ns.ljust(9, "0")  # pad to 9 digits for nanoseconds
+                    stamp = Stamp(sec=int(s), nsec=int(ns))
+                elif "sec" not in fieldnames:
+                    stamp = Stamp.from_nsec(int(line["nsec"]))
+                else:
+                    stamp = Stamp(sec=int(line["sec"]), nsec=int(line["nsec"]))
+                poses.append(pose)
+                stamps.append(stamp)
+
+        return Trajectory(stamps=stamps, poses=poses)
+
+    @staticmethod
+    def from_tum(path: Path) -> "Trajectory":
+        """Load a TUM dataset pose file. Simple wrapper around [from_csv][evalio.types.Trajectory].
+
+        Args:
+            path (Path): Location of file.
+
+        Returns:
+            Trajectory: Stored trajectory
+        """
+        return Trajectory.from_csv(path, ["sec", "x", "y", "z", "qx", "qy", "qz", "qw"])
+
+    @staticmethod
+    def from_file(path: Path) -> Trajectory | FailedMetadataParse:
+        """Load a saved evalio trajectory from file.
+
+        Works identically to [from_tum][evalio.types.Trajectory.from_tum], but also loads metadata from the file.
+
+        Args:
+            path (Path): Location of trajectory results.
+
+        Returns:
+            Trajectory: Loaded trajectory with metadata, stamps, and poses.
+        """
+        with open(path) as file:
+            metadata_filter = filter(
+                lambda row: row[0] == "#" and not row.startswith("# timestamp,"), file
+            )
+            metadata_list = [row[1:].strip() for row in metadata_filter]
+            metadata_str = "\n".join(metadata_list)
+
+            metadata = Metadata.parse(metadata_str)
+            if isinstance(metadata, FailedMetadataParse):
+                return metadata
+
+            metadata.file = path
+
+        trajectory = Trajectory.from_csv(
+            path,
+            fieldnames=["sec", "x", "y", "z", "qx", "qy", "qz", "qw"],
+        )
+        trajectory.metadata = metadata
+
+        return trajectory
+
+    # ------------------------- Saving to file ------------------------- #
+    def _serialize_pose(self, stamp: Stamp, pose: SE3) -> list[str | float]:
+        """Helper to serialize a stamped pose for csv writing.
+
+        Args:
+            stamp (Stamp): Timestamp associated with the pose.
+            pose (SE3): Pose to save.
+        """
+        return [
+            f"{stamp.sec}.{stamp.nsec:09}",
+            pose.trans[0],
+            pose.trans[1],
+            pose.trans[2],
+            pose.rot.qx,
+            pose.rot.qy,
+            pose.rot.qz,
+            pose.rot.qw,
+        ]
+
+    def _serialize_metadata(self) -> str:
+        if self.metadata is None:
+            return ""
+
+        metadata_str = self.metadata.to_yaml()
+        metadata_str = metadata_str.replace("\n", "\n# ")
+        return f"# {metadata_str}\n#\n"
+
+    def open(self, path: Path):
+        self._file = path.open("w")
+        self._csv_writer = csv.writer(self._file)
+
+        # write everything we've got so far
+        if self.metadata is not None:
+            self._file.write(self._serialize_metadata())
+
+        self._file.write("# timestamp, x, y, z, qx, qy, qz, qw\n")
+        self._csv_writer.writerow(self._serialize_pose(s, p) for s, p in self)
+
+    def close(self):
+        """Close the CSV file if it is open with [write_experiment][evalio.types.Trajectory.write_experiment] and incremental writing."""
+        if self._file is not None:
+            self._file.close()
+            self._file = None
+            self._csv_writer = None
+        else:
+            print_warning("Trajectory.close: No file to close.")
+
+    def to_file(self, path: Path):
+        self.open(path)
+        self.close()
+
+    def update_metadata(self):
+        """Update the metadata in an open file."""
+        if self._file is None or self._csv_writer is None:
+            print_warning("Trajectory.update_metadata: No file is open.")
+            return
+
+        if self.metadata is None:
+            print_warning("Trajectory.update_metadata: No metadata to update.")
+            return
+
+        # Go back to the start of the file and rewrite the metadata
+        self._file.seek(0)
+        self._file.write(self._serialize_metadata())
+
+        # Rewrite all the poses
+        self._file.write("# timestamp, x, y, z, qx, qy, qz, qw\n")
+        self._csv_writer.writerow(self._serialize_pose(s, p) for s, p in self)
diff --git a/python/evalio/types/extended.py b/python/evalio/types/extended.py
new file mode 100644
index 00000000..20fce46d
--- /dev/null
+++ b/python/evalio/types/extended.py
@@ -0,0 +1,123 @@
+"""
+These are extended types that do depend on other parts of evalio.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from dataclasses import dataclass, InitVar
+from typing import Any, Optional, Self
+from evalio.types.base import Param, Metadata
+
+from evalio import pipelines as pl, datasets as ds
+from evalio.utils import print_warning
+
+
+class ExperimentStatus(Enum):
+    Complete = "complete"
+    Fail = "fail"
+    Started = "started"
+
+
+@dataclass(kw_only=True)
+class Experiment(Metadata):
+    name: str
+    """Name of the experiment."""
+    sequence: str
+    """Dataset used to run the experiment."""
+    sequence_length: int
+    """Length of the sequence, if set"""
+    pipeline: str
+    """Pipeline used to generate the trajectory."""
+    pipeline_version: str
+    """Version of the pipeline used."""
+    pipeline_params: dict[str, Param]
+    """Parameters used for the pipeline."""
+    status: ExperimentStatus = ExperimentStatus.Started
+    """Status of the experiment, e.g. "success", "failure", etc."""
+    total_elapsed: Optional[float] = None
+    """Total time taken for the experiment, as a string."""
+    max_step_elapsed: Optional[float] = None
+    """Maximum time taken for a single step in the experiment, as a string."""
+    do_verify: InitVar[bool] = False
+    """If true, verify the experiment parameters are valid."""
+
+    def __post_init__(self, do_verify: bool):
+        if do_verify:
+            self.verify()
+
+    def verify(self):
+        # Verify pipeline is good
+        ThisPipeline = pl.get_pipeline(self.pipeline)
+        if isinstance(ThisPipeline, pl.PipelineNotFound):
+            raise ValueError(
+                f"Experiment '{self.name}' has unknown pipeline '{self.pipeline}'"
+            )
+
+        all_params = ThisPipeline.default_params()
+        for key in self.pipeline_params.keys():
+            if key not in all_params:
+                raise ValueError(
+                    f"Invalid parameter '{key}' for pipeline '{ThisPipeline.name()}'"
+                )
+            elif key in all_params and not isinstance(
+                self.pipeline_params[key], type(all_params[key])
+            ):
+                raise ValueError(
+                    f"Invalid type for parameter '{key}' for pipeline '{ThisPipeline.name()}': "
+                    f"expected '{type(all_params[key]).__name__}', got '{type(self.pipeline_params[key]).__name__}'"
+                )
+
+        # Verify dataset is good
+        dataset = ds.get_sequence(self.sequence)
+        if isinstance(dataset, ds.SequenceNotFound):
+            raise ValueError(
+                f"Experiment {self.name} has unknown dataset {self.sequence}"
+            )
+
+        if self.sequence_length > (length := len(dataset)):
+            print_warning(
+                f"Experiment '{self.name}' has sequence_length {self.sequence_length} > dataset length {len(dataset)}, reducing to {len(dataset)}"
+            )
+            self.sequence_length = length
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> Self:
+        if "status" in data:
+            data["status"] = ExperimentStatus(data["status"])
+
+        return super().from_dict(data)
+
+    def make_pipeline(self) -> pl.Pipeline:
+        ThisPipeline = pl.get_pipeline(self.pipeline)
+        if isinstance(ThisPipeline, pl.PipelineNotFound):
+            raise ValueError(
+                f"Experiment {self.name} has unknown pipeline {self.pipeline}"
+            )
+
+        dataset = ds.get_sequence(self.sequence)
+        if isinstance(dataset, ds.SequenceNotFound):
+            raise ValueError(
+                f"Experiment {self.name} has unknown dataset {self.sequence}"
+            )
+
+        pipe = ThisPipeline()
+
+        # Set user params
+        params = pipe.set_params(self.pipeline_params)
+        if len(params) > 0:
+            for k, v in params.items():
+                print_warning(
+                    f"Pipeline {self.name} has unused parameters: {k}={v}. "
+                    "Please check your configuration."
+                )
+
+        # Set dataset params
+        pipe.set_imu_params(dataset.imu_params())
+        pipe.set_lidar_params(dataset.lidar_params())
+        pipe.set_imu_T_lidar(dataset.imu_T_lidar())
+
+        # Initialize pipeline
+        pipe.initialize()
+
+        return pipe
diff --git a/python/evalio/utils.py b/python/evalio/utils.py
index df555a0c..19625054 100644
--- a/python/evalio/utils.py
+++ b/python/evalio/utils.py
@@ -1,3 +1,4 @@
+from enum import Enum, auto
 from rich.console import Console
 
 
@@ -6,3 +7,44 @@ def print_warning(warn: str):
     Print a warning message.
     """
     Console(soft_wrap=True).print(f"[bold red]Warning[/bold red]: {warn}")
+
+
+# For converting dataset names to snake case
+class CharKinds(Enum):
+    LOWER = auto()
+    UPPER = auto()
+    DIGIT = auto()
+    OTHER = auto()
+
+    @staticmethod
+    def from_char(char: str):
+        if char.islower():
+            return CharKinds.LOWER
+        if char.isupper():
+            return CharKinds.UPPER
+        if char.isdigit():
+            return CharKinds.DIGIT
+        return CharKinds.OTHER
+
+
+def pascal_to_snake(identifier: str) -> str:
+    """Convert a PascalCase identifier to snake_case.
+
+    Args:
+        identifier (str): The PascalCase identifier to convert.
+
+    Returns:
+        str: The converted snake_case identifier.
+    """
+    # Only split when going from lower to something else
+    # this handles digits better than other approaches
+    splits: list[int] = []
+    last_kind = CharKinds.from_char(identifier[0])
+    for i, char in enumerate(identifier[1:], start=1):
+        kind = CharKinds.from_char(char)
+        if last_kind == CharKinds.LOWER and kind != CharKinds.LOWER:
+            splits.append(i)
+        last_kind = kind
+
+    parts = [identifier[i:j] for i, j in zip([0] + splits, splits + [None])]
+    return "_".join(parts).lower()
diff --git a/uv.lock b/uv.lock
index b0df1979..644f69a0 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1632,11 +1632,11 @@ wheels = [
 
 [[package]]
 name = "typing-extensions"
-version = "4.13.2"
+version = "4.12.2"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/f6/37/23083fcd6e35492953e8d2aaaa68b860eb422b34627b13f2ce3eb6106061/typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef", size = 106967, upload-time = "2025-04-10T14:19:05.416Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8", size = 85321, upload-time = "2024-06-07T18:52:15.995Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/8b/54/b1ae86c0973cc6f0210b53d508ca3641fb6d0c56823f288d108bc7ab3cc8/typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c", size = 45806, upload-time = "2025-04-10T14:19:03.967Z" },
+    { url = "https://files.pythonhosted.org/packages/26/9f/ad63fc0248c5379346306f8668cda6e2e2e9c95e01216d2b8ffd9ff037d0/typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", size = 37438, upload-time = "2024-06-07T18:52:13.582Z" },
 ]
 
 [[package]]

From d4ffdd099eb78a6a0de109a82c4cbb9d470c802d Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Tue, 30 Sep 2025 06:49:37 -0400
Subject: [PATCH 05/40] Finalizing all parsing code

---
 python/evalio/datasets/__init__.py  |  2 +
 python/evalio/datasets/parser.py    | 23 ++++----
 python/evalio/pipelines/__init__.py | 10 ++++
 python/evalio/pipelines/parser.py   | 72 ++++++++++++++++++++----
 python/evalio/utils.py              |  7 +++
 tests/test_parsing.py               | 87 +++++++++++++++++++++++++++++
 6 files changed, 180 insertions(+), 21 deletions(-)
 create mode 100644 tests/test_parsing.py

diff --git a/python/evalio/datasets/__init__.py b/python/evalio/datasets/__init__.py
index 81c14aa9..a1f5a658 100644
--- a/python/evalio/datasets/__init__.py
+++ b/python/evalio/datasets/__init__.py
@@ -20,6 +20,7 @@
     DatasetNotFound,
     SequenceNotFound,
     InvalidDatasetConfig,
+    DatasetConfigError,
 )
 
 __all__ = [
@@ -47,4 +48,5 @@
     "DatasetNotFound",
     "SequenceNotFound",
     "InvalidDatasetConfig",
+    "DatasetConfigError",
 ]
diff --git a/python/evalio/datasets/parser.py b/python/evalio/datasets/parser.py
index e4472173..10df64e8 100644
--- a/python/evalio/datasets/parser.py
+++ b/python/evalio/datasets/parser.py
@@ -2,32 +2,36 @@
 from inspect import isclass
 import itertools
 from types import ModuleType
-from typing import Callable, Optional, Sequence, TypedDict, cast
+from typing import Callable, NotRequired, Optional, Sequence, TypedDict, cast
 
 from evalio import datasets
 from evalio.datasets.base import Dataset
+from evalio.utils import CustomException
 
 _DATASETS: set[type[Dataset]] = set()
 
 
-class DatasetNotFound(Exception):
+class DatasetNotFound(CustomException):
     def __init__(self, name: str):
         super().__init__(f"Dataset '{name}' not found")
         self.name = name
 
 
-class SequenceNotFound(Exception):
+class SequenceNotFound(CustomException):
     def __init__(self, name: str):
         super().__init__(f"Sequence '{name}' not found")
         self.name = name
 
 
-class InvalidDatasetConfig(Exception):
+class InvalidDatasetConfig(CustomException):
     def __init__(self, config: str):
         super().__init__(f"Invalid config: '{config}'")
         self.config = config
 
 
+DatasetConfigError = DatasetNotFound | SequenceNotFound | InvalidDatasetConfig
+
+
 # ------------------------- Handle Registration of Datasets ------------------------- #
 def _is_dataset(obj: object) -> bool:
     return (
@@ -91,22 +95,19 @@ def get_sequence(name: str) -> Dataset | SequenceNotFound:
 # ------------------------- Handle yaml parsing ------------------------- #
 class DatasetConfig(TypedDict):
     name: str
-    length: Optional[int]
-
-
-ConfigError = DatasetNotFound | SequenceNotFound | InvalidDatasetConfig
+    length: NotRequired[Optional[int]]
 
 
 def parse_config(
     d: str | DatasetConfig | Sequence[str | DatasetConfig],
-) -> list[tuple[Dataset, int]] | ConfigError:
+) -> list[tuple[Dataset, int]] | DatasetConfigError:
     name: Optional[str] = None
     length: Optional[int] = None
     # If given a list of values
     if isinstance(d, list):
         results = [parse_config(x) for x in d]
         for r in results:
-            if isinstance(r, ConfigError):
+            if isinstance(r, DatasetConfigError):
                 return r
         results = cast(list[list[tuple[Dataset, int]]], results)
         return list(itertools.chain.from_iterable(results))
@@ -122,7 +123,7 @@ def parse_config(
         return InvalidDatasetConfig(str(d))
 
     if name is None:  # type: ignore
-        return InvalidDatasetConfig(str(d))
+        return InvalidDatasetConfig("Missing 'name' in dataset config")
 
     length_lambda: Callable[[Dataset], int]
     if length is None:
diff --git a/python/evalio/pipelines/__init__.py b/python/evalio/pipelines/__init__.py
index d5b72462..6c912de3 100644
--- a/python/evalio/pipelines/__init__.py
+++ b/python/evalio/pipelines/__init__.py
@@ -4,8 +4,13 @@
     register_pipeline,
     get_pipeline,
     all_pipelines,
+    parse_config,
     PipelineNotFound,
     InvalidPipelineConfig,
+    PipelineConfigError,
+    UnusedPipelineParam,
+    InvalidPipelineParamType,
+    validate_params,
 )
 
 
@@ -13,6 +18,11 @@
     "all_pipelines",
     "get_pipeline",
     "register_pipeline",
+    "parse_config",
+    "validate_params",
     "PipelineNotFound",
     "InvalidPipelineConfig",
+    "UnusedPipelineParam",
+    "InvalidPipelineParamType",
+    "PipelineConfigError",
 ]
diff --git a/python/evalio/pipelines/parser.py b/python/evalio/pipelines/parser.py
index a62ea286..aeb76504 100644
--- a/python/evalio/pipelines/parser.py
+++ b/python/evalio/pipelines/parser.py
@@ -9,23 +9,49 @@
 from evalio import pipelines
 from evalio.pipelines import Pipeline
 from evalio.types import Param
+from evalio.utils import CustomException
 
 
 _PIPELINES: set[type[Pipeline]] = set()
 
 
-class PipelineNotFound(Exception):
+class PipelineNotFound(CustomException):
     def __init__(self, name: str):
         super().__init__(f"Pipeline '{name}' not found")
         self.name = name
 
 
-class InvalidPipelineConfig(Exception):
+class InvalidPipelineConfig(CustomException):
     def __init__(self, config: str):
         super().__init__(f"Invalid config: '{config}'")
         self.config = config
 
 
+class UnusedPipelineParam(CustomException):
+    def __init__(self, param: str, pipeline: str):
+        super().__init__(f"Parameter '{param}' is not used in pipeline '{pipeline}'")
+        self.param = param
+        self.pipeline = pipeline
+
+
+class InvalidPipelineParamType(CustomException):
+    def __init__(self, param: str, expected_type: type, actual_type: type):
+        super().__init__(
+            f"Parameter '{param}' has invalid type. Expected '{expected_type.__name__}', got '{actual_type.__name__}'"
+        )
+        self.param = param
+        self.expected_type = expected_type
+        self.actual_type = actual_type
+
+
+PipelineConfigError = (
+    PipelineNotFound
+    | InvalidPipelineConfig
+    | UnusedPipelineParam
+    | InvalidPipelineParamType
+)
+
+
 # ------------------------- Handle Registration of Pipelines ------------------------- #
 def _is_pipe(obj: Any) -> bool:
     return (
@@ -103,32 +129,54 @@ def _sweep(
     sweep: dict[str, Param],
     params: dict[str, Param],
     pipe: type[Pipeline],
-) -> list[tuple[type[Pipeline], dict[str, Param]]]:
+) -> list[tuple[type[Pipeline], dict[str, Param]]] | PipelineConfigError:
     keys, values = zip(*sweep.items())
     results: list[tuple[type[Pipeline], dict[str, Param]]] = []
     for options in itertools.product(*values):
         p = params.copy()
         for k, o in zip(keys, options):
             p[k] = o
+        err = validate_params(pipe, p)
+        if err is not None:
+            return err
         results.append((pipe, p))
     return results
 
 
-ConfigError = PipelineNotFound | InvalidPipelineConfig
+def validate_params(
+    pipe: type[Pipeline],
+    params: dict[str, Param],
+) -> None | PipelineConfigError:
+    """Validate the parameters for a given pipeline.
+
+    Args:
+        pipe (type[Pipeline]): The pipeline class.
+        params (dict[str, Param]): The parameters to validate.
+
+    Returns:
+        Optional[PipelineConfigError]: An error if validation fails, otherwise None.
+    """
+    default_params = pipe.default_params()
+    for p in params:
+        if p not in default_params:
+            return UnusedPipelineParam(p, pipe.name())
+
+        expected_type = type(default_params[p])
+        actual_type = type(params[p])
+        if actual_type != expected_type:
+            return InvalidPipelineParamType(p, expected_type, actual_type)
+
+    return None
 
 
 def parse_config(
     p: str | dict[str, Param] | Sequence[str | dict[str, Param]],
-) -> list[tuple[type[Pipeline], dict[str, Param]]] | ConfigError:
+) -> list[tuple[type[Pipeline], dict[str, Param]]] | PipelineConfigError:
     """Parse a pipeline configuration.
 
     Args:
         p (str | dict[str, Param] | Sequence[str | dict[str, Param]]): The pipeline configuration.
 
-    Raises:
-        ValueError: If the pipeline is not found.
-        ValueError: If the configuration is invalid.
-
     Returns:
         list[tuple[type[Pipeline], dict[str, Param]]]: A list of tuples containing the pipeline class and its parameters.
     """
@@ -151,12 +199,16 @@ def parse_config(
             sweep = cast(dict[str, Param], p.pop("sweep"))
             return _sweep(sweep, p, pipe)
         else:
+            err = validate_params(pipe, p)
+            if err is not None:
+                return err
+
             return [(pipe, p)]
 
     elif isinstance(p, list):
         results = [parse_config(x) for x in p]
         for r in results:
-            if isinstance(r, ConfigError):
+            if isinstance(r, PipelineConfigError):
                 return r
         results = cast(list[list[tuple[type[Pipeline], dict[str, Param]]]], results)
         return list(itertools.chain.from_iterable(results))
diff --git a/python/evalio/utils.py b/python/evalio/utils.py
index 19625054..bd1621d6 100644
--- a/python/evalio/utils.py
+++ b/python/evalio/utils.py
@@ -9,6 +9,13 @@ def print_warning(warn: str):
     Console(soft_wrap=True).print(f"[bold red]Warning[/bold red]: {warn}")
 
 
+class CustomException(Exception):
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, self.__class__):
+            return False
+        return self.args == other.args
+
+
 # For converting dataset names to snake case
 class CharKinds(Enum):
     LOWER = auto()
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
new file mode 100644
index 00000000..ec9e639c
--- /dev/null
+++ b/tests/test_parsing.py
@@ -0,0 +1,87 @@
+import evalio.datasets as ds
+from evalio.types import Param
+from evalio.datasets.parser import DatasetConfig, DatasetConfigError
+import evalio.pipelines as pl
+from typing import Any, Sequence
+import pytest
+
+# ------------------------- Dataset Parsing ------------------------- #
+seq = ds.NewerCollege2021.quad_easy
+name = seq.full_name
+
+# fmt: off
+DATASETS: list[tuple[str | DatasetConfig | Sequence[str | DatasetConfig], DatasetConfigError | list[tuple[ds.Dataset, int]]]] = [
+    # good ones
+    (name, [(seq, len(seq))]),
+    ({"name": name}, [(seq, len(seq))]),
+    ({"name": name, "length": 100}, [(seq, 100)]),
+    ({"name": f"{seq.dataset_name()}/*"}, [(s, len(s)) for s in seq.sequences()]),
+    # bad ones
+    ("newer_college_2020/bad", ds.SequenceNotFound("newer_college_2020/bad")),
+    ("newer_college_123/*", ds.DatasetNotFound(name="newer_college_123")),
+    ("newer_college_123/*", ds.DatasetNotFound(name="newer_college_123")),
+    ({"name": "newer_college_2020/bad"}, ds.SequenceNotFound("newer_college_2020/bad")),
+    ({"length": 100}, ds.InvalidDatasetConfig("Missing 'name' in dataset config")), # type: ignore
+]
+# fmt: on
+
+
+# Test to ensure datasets are parsed correctly
+@pytest.mark.parametrize("dataset_name, expected", DATASETS)
+def test_dataset_parsing(
+    dataset_name: str | DatasetConfig | Sequence[str | DatasetConfig],
+    expected: DatasetConfigError | list[tuple[ds.Dataset, int]],
+):
+    dataset = ds.parse_config(dataset_name)
+    assert dataset == expected
+
+
+# ------------------------- Pipeline Parsing ------------------------- #
+class FakePipeline(pl.Pipeline):
+    @staticmethod
+    def name() -> str:
+        return "fake"
+
+    @staticmethod
+    def version() -> str:
+        return "0.1.0"
+
+    @staticmethod
+    def default_params() -> dict[str, Param]:
+        return {"param1": 1, "param2": "value"}
+
+
+pl.register_pipeline(FakePipeline)
+
+# fmt: off
+PIPELINES: list[Any] = [
+    # good ones
+    ("fake", [(FakePipeline, {})]),
+    ({"name": "fake"}, [(FakePipeline, {})]),
+    ({"name": "fake", "param1": 5}, [(FakePipeline, {"param1": 5})]),
+    (["fake", {"name": "fake", "param1": 3}], [(FakePipeline, {}), (FakePipeline, {"param1": 3})]),
+    ({"name": "fake", "sweep": {"param1": [1, 2, 3]}}, [
+        (FakePipeline, {"param1": 1}),
+        (FakePipeline, {"param1": 2}),
+        (FakePipeline, {"param1": 3}),
+    ]),
+    # bad ones
+    ("unknown", pl.PipelineNotFound("unknown")),
+    ({"name": "unknown"}, pl.PipelineNotFound("unknown")),
+    ({"param1": 5}, pl.InvalidPipelineConfig("Need pipeline name: {'param1': 5}")), # type: ignore
+    ({"name": "fake", "param3": 10}, pl.UnusedPipelineParam("param3", "fake")),
+    ({"name": "fake", "param1": "wrong_type"}, pl.InvalidPipelineParamType("param1", int, str)),
+    ({"name": "fake", "sweep": {"param1": [1.0, 2, 3]}}, pl.InvalidPipelineParamType("param1", int, float)),
+    ({"name": "fake", "sweep": {"param3": [1.0, 2, 3]}}, pl.UnusedPipelineParam("param3", "fake")),
+]
+# fmt: on
+
+
+# Test to ensure pipelines are parsed correctly
+@pytest.mark.parametrize("pipeline_name, expected", PIPELINES)
+def test_pipeline_parsing(
+    pipeline_name: str | dict[str, Param] | Sequence[str | dict[str, Param]],
+    expected: list[tuple[type[pl.Pipeline], dict[str, Param]]] | pl.PipelineConfigError,
+):
+    pipeline = pl.parse_config(pipeline_name)
+    assert pipeline == expected

From 261f165d5160be62bf0e72dcbaafd48e0cdb876f Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Tue, 30 Sep 2025 11:30:45 -0400
Subject: [PATCH 06/40] Finish adding tests for new trajectory and metadata
 classes

---
 python/evalio/datasets/parser.py              | 34 +++----
 python/evalio/pipelines/parser.py             | 29 +++---
 python/evalio/types/base.py                   | 51 +++++-----
 python/evalio/types/extended.py               | 58 ++----------
 ...dataset_loading.py => test_csv_loading.py} |  0
 tests/test_experiment.py                      | 92 -------------------
 tests/test_io.py                              | 79 ++++++++++++++++
 7 files changed, 145 insertions(+), 198 deletions(-)
 rename tests/{test_dataset_loading.py => test_csv_loading.py} (100%)
 delete mode 100644 tests/test_experiment.py
 create mode 100644 tests/test_io.py

diff --git a/python/evalio/datasets/parser.py b/python/evalio/datasets/parser.py
index 10df64e8..9c3e66f6 100644
--- a/python/evalio/datasets/parser.py
+++ b/python/evalio/datasets/parser.py
@@ -46,28 +46,26 @@ def _search_module(module: ModuleType) -> set[type[Dataset]]:
 def register_dataset(
     dataset: Optional[type[Dataset]] = None,
     module: Optional[ModuleType | str] = None,
-):
+) -> int | ImportError:
     global _DATASETS
 
+    total = 0
     if module is not None:
         if isinstance(module, str):
             try:
                 module = importlib.import_module(module)
-            except ImportError:
-                raise ValueError(f"Failed to import '{module}'")
+            except ImportError as e:
+                return e
 
-        if len(new_ds := _search_module(module)) > 0:
-            _DATASETS.update(new_ds)
-        else:
-            raise ValueError(
-                f"Module {module.__name__} does not contain any datasets or pipelines"
-            )
+        new_ds = _search_module(module)
+        _DATASETS.update(new_ds)
+        total += len(new_ds)
 
-    if dataset is not None:
-        if _is_dataset(dataset):
-            _DATASETS.add(dataset)
-        else:
-            raise ValueError(f"{dataset} is not a valid Dataset subclass")
+    if dataset is not None and _is_dataset(dataset):
+        _DATASETS.add(dataset)
+        total += 1
+
+    return total
 
 
 def all_datasets() -> dict[str, type[Dataset]]:
@@ -125,11 +123,9 @@ def parse_config(
     if name is None:  # type: ignore
         return InvalidDatasetConfig("Missing 'name' in dataset config")
 
-    length_lambda: Callable[[Dataset], int]
-    if length is None:
-        length_lambda = lambda s: len(s)
-    else:
-        length_lambda = lambda s: length
+    length_lambda: Callable[[Dataset], int] = (
+        lambda s: len(s) if length is None else min(len(s), length)
+    )
 
     if name[-2:] == "/*":
         ds_name, _ = name.split("/")
diff --git a/python/evalio/pipelines/parser.py b/python/evalio/pipelines/parser.py
index aeb76504..bec54421 100644
--- a/python/evalio/pipelines/parser.py
+++ b/python/evalio/pipelines/parser.py
@@ -66,37 +66,32 @@ def _search_module(module: ModuleType) -> set[type[Pipeline]]:
 def register_pipeline(
     pipeline: Optional[type[Pipeline]] = None,
     module: Optional[ModuleType | str] = None,
-):
+) -> int | ImportError:
     """Add a pipeline or a module containing pipelines to the registry.
 
     Args:
         pipeline (Optional[type[Pipeline]], optional): A specific pipeline class to add. Defaults to None.
         module (Optional[ModuleType  |  str], optional): The module to search for pipelines. Defaults to None.
-
-    Raises:
-        ValueError: If the module does not contain any pipelines.
-        ValueError: If the pipeline is not a valid Pipeline subclass.
-        ValueError: If both module and pipeline are None.
     """
     global _PIPELINES
 
+    total = 0
     if module is not None:
         if isinstance(module, str):
             try:
                 module = importlib.import_module(module)
-            except ImportError:
-                raise ValueError(f"Failed to import '{module}'")
+            except ImportError as e:
+                return e
 
-        if len(new_pipes := _search_module(module)) > 0:
-            _PIPELINES.update(new_pipes)
-        else:
-            raise ValueError(f"Module {module.__name__} does not contain any pipelines")
+        new_pipes = _search_module(module)
+        _PIPELINES.update(new_pipes)
+        total += len(new_pipes)
 
-    if pipeline is not None:
-        if _is_pipe(pipeline):
-            _PIPELINES.add(pipeline)
-        else:
-            raise ValueError(f"{pipeline} is not a valid Pipeline subclass")
+    if pipeline is not None and _is_pipe(pipeline):
+        _PIPELINES.add(pipeline)
+        total += 1
+
+    return total
 
 
 def all_pipelines() -> dict[str, type[Pipeline]]:
diff --git a/python/evalio/types/base.py b/python/evalio/types/base.py
index 2a00059f..abfbf3af 100644
--- a/python/evalio/types/base.py
+++ b/python/evalio/types/base.py
@@ -61,18 +61,20 @@ def from_dict(cls, data: dict[str, Any]) -> Self:
         return cls(**data)
 
     def to_dict(self) -> dict[str, Any]:
-        return asdict(self)
+        d = asdict(self)
+        d["type"] = self.tag()  # add type tag for deserialization
+        del d["file"]  # don't serialize the file path
+        return d
 
     def to_yaml(self) -> str:
         data = self.to_dict()
-        data["type"] = self.tag()
         return yaml.safe_dump(data)
 
     @classmethod
-    def parse(cls, yaml_str: str) -> Metadata | FailedMetadataParse:
+    def from_yaml(cls, yaml_str: str) -> Metadata | FailedMetadataParse:
         data = yaml.safe_load(yaml_str)
 
-        if "type: " not in data:
+        if "type" not in data:
             return FailedMetadataParse("No type field found in metadata.")
 
         for name, subclass in cls._registry.items():
@@ -218,10 +220,10 @@ def from_file(path: Path) -> Trajectory | FailedMetadataParse:
             metadata_filter = filter(
                 lambda row: row[0] == "#" and not row.startswith("# timestamp,"), file
             )
-            metadata_list = [row[1:].strip() for row in metadata_filter]
-            metadata_str = "\n".join(metadata_list)
+            metadata_list = [row[1:] for row in metadata_filter]
+            metadata_str = "".join(metadata_list)
 
-            metadata = Metadata.parse(metadata_str)
+            metadata = Metadata.from_yaml(metadata_str)
             if isinstance(metadata, FailedMetadataParse):
                 return metadata
 
@@ -260,18 +262,26 @@ def _serialize_metadata(self) -> str:
 
         metadata_str = self.metadata.to_yaml()
         metadata_str = metadata_str.replace("\n", "\n# ")
-        return f"# {metadata_str}\n#\n"
+        return f"# {metadata_str}\n"
 
-    def open(self, path: Path):
-        self._file = path.open("w")
-        self._csv_writer = csv.writer(self._file)
+    def _write(self):
+        if self._file is None or self._csv_writer is None:
+            print_warning("Trajectory.write_experiment: No file is open.")
+            return
 
         # write everything we've got so far
         if self.metadata is not None:
             self._file.write(self._serialize_metadata())
 
         self._file.write("# timestamp, x, y, z, qx, qy, qz, qw\n")
-        self._csv_writer.writerow(self._serialize_pose(s, p) for s, p in self)
+        self._csv_writer.writerows(self._serialize_pose(s, p) for s, p in self)
+
+    def open(self, path: Path):
+        if self.metadata is not None:
+            self.metadata.file = path
+        self._file = path.open("w")
+        self._csv_writer = csv.writer(self._file)
+        self._write()
 
     def close(self):
         """Close the CSV file if it is open with [write_experiment][evalio.types.Trajectory.write_experiment] and incremental writing."""
@@ -286,20 +296,17 @@ def to_file(self, path: Path):
         self.open(path)
         self.close()
 
-    def update_metadata(self):
-        """Update the metadata in an open file."""
+    def rewrite(self):
+        """Update the contents of an open file."""
         if self._file is None or self._csv_writer is None:
-            print_warning("Trajectory.update_metadata: No file is open.")
+            print_warning("Trajectory.rewrite: No file is open.")
             return
 
         if self.metadata is None:
-            print_warning("Trajectory.update_metadata: No metadata to update.")
+            print_warning("Trajectory.rewrite: No metadata to update.")
             return
 
-        # Go back to the start of the file and rewrite the metadata
+        # Go to start, empty, and rewrite
         self._file.seek(0)
-        self._file.write(self._serialize_metadata())
-
-        # Rewrite all the poses
-        self._file.write("# timestamp, x, y, z, qx, qy, qz, qw\n")
-        self._csv_writer.writerow(self._serialize_pose(s, p) for s, p in self)
+        self._file.truncate()
+        self._write()
diff --git a/python/evalio/types/extended.py b/python/evalio/types/extended.py
index 20fce46d..7cecea45 100644
--- a/python/evalio/types/extended.py
+++ b/python/evalio/types/extended.py
@@ -5,7 +5,7 @@
 from __future__ import annotations
 
 from enum import Enum
-from dataclasses import dataclass, InitVar
+from dataclasses import dataclass
 from typing import Any, Optional, Self
 from evalio.types.base import Param, Metadata
 
@@ -39,47 +39,11 @@ class Experiment(Metadata):
     """Total time taken for the experiment, as a string."""
     max_step_elapsed: Optional[float] = None
     """Maximum time taken for a single step in the experiment, as a string."""
-    do_verify: InitVar[bool] = False
-    """If true, verify the experiment parameters are valid."""
 
-    def __post_init__(self, do_verify: bool):
-        if do_verify:
-            self.verify()
-
-    def verify(self):
-        # Verify pipeline is good
-        ThisPipeline = pl.get_pipeline(self.pipeline)
-        if isinstance(ThisPipeline, pl.PipelineNotFound):
-            raise ValueError(
-                f"Experiment '{self.name}' has unknown pipeline '{self.pipeline}'"
-            )
-
-        all_params = ThisPipeline.default_params()
-        for key in self.pipeline_params.keys():
-            if key not in all_params:
-                raise ValueError(
-                    f"Invalid parameter '{key}' for pipeline '{ThisPipeline.name()}'"
-                )
-            elif key in all_params and not isinstance(
-                self.pipeline_params[key], type(all_params[key])
-            ):
-                raise ValueError(
-                    f"Invalid type for parameter '{key}' for pipeline '{ThisPipeline.name()}': "
-                    f"expected '{type(all_params[key]).__name__}', got '{type(self.pipeline_params[key]).__name__}'"
-                )
-
-        # Verify dataset is good
-        dataset = ds.get_sequence(self.sequence)
-        if isinstance(dataset, ds.SequenceNotFound):
-            raise ValueError(
-                f"Experiment {self.name} has unknown dataset {self.sequence}"
-            )
-
-        if self.sequence_length > (length := len(dataset)):
-            print_warning(
-                f"Experiment '{self.name}' has sequence_length {self.sequence_length} > dataset length {len(dataset)}, reducing to {len(dataset)}"
-            )
-            self.sequence_length = length
+    def to_dict(self) -> dict[str, Any]:
+        d = super().to_dict()
+        d["status"] = self.status.value
+        return d
 
     @classmethod
     def from_dict(cls, data: dict[str, Any]) -> Self:
@@ -88,18 +52,16 @@ def from_dict(cls, data: dict[str, Any]) -> Self:
 
         return super().from_dict(data)
 
-    def make_pipeline(self) -> pl.Pipeline:
+    def make_pipeline(
+        self,
+    ) -> pl.Pipeline | ds.DatasetConfigError | pl.PipelineConfigError:
         ThisPipeline = pl.get_pipeline(self.pipeline)
         if isinstance(ThisPipeline, pl.PipelineNotFound):
-            raise ValueError(
-                f"Experiment {self.name} has unknown pipeline {self.pipeline}"
-            )
+            return ThisPipeline
 
         dataset = ds.get_sequence(self.sequence)
         if isinstance(dataset, ds.SequenceNotFound):
-            raise ValueError(
-                f"Experiment {self.name} has unknown dataset {self.sequence}"
-            )
+            return dataset
 
         pipe = ThisPipeline()
 
diff --git a/tests/test_dataset_loading.py b/tests/test_csv_loading.py
similarity index 100%
rename from tests/test_dataset_loading.py
rename to tests/test_csv_loading.py
diff --git a/tests/test_experiment.py b/tests/test_experiment.py
deleted file mode 100644
index 3009636e..00000000
--- a/tests/test_experiment.py
+++ /dev/null
@@ -1,92 +0,0 @@
-from evalio.types import Experiment, ExperimentStatus, Param
-from evalio.pipelines import Pipeline, register_pipeline
-
-import pytest
-
-
-class FakePipeline(Pipeline):
-    @staticmethod
-    def name() -> str:
-        return "fake"
-
-    @staticmethod
-    def version() -> str:
-        return "0.1.0"
-
-    @staticmethod
-    def default_params() -> dict[str, Param]:
-        return {"param1": 1, "param2": "value"}
-
-
-def test_serde():
-    exp = Experiment(
-        name="test",
-        status=ExperimentStatus.Complete,
-        sequence="newer_college_2020/short_experiment",
-        sequence_length=1000,
-        pipeline="fake",
-        pipeline_version="0.1.0",
-        pipeline_params={"param1": 1, "param2": "value"},
-        total_elapsed=10.5,
-        max_step_elapsed=0.24,
-    )
-    out = Experiment.from_yaml(exp.to_yaml())
-    assert exp == out
-
-
-def test_verify(capsys: pytest.CaptureFixture[str]):
-    register_pipeline(FakePipeline)
-    misc = {
-        "name": "test",
-        "status": ExperimentStatus.Complete,
-        "sequence": "newer_college_2020/short_experiment",
-        "pipeline_version": "0.1.0",
-        "do_verify": True,
-    }
-
-    # Bad pipeline name
-    with pytest.raises(ValueError) as exc:
-        Experiment(
-            **misc,  # type: ignore
-            sequence_length=1000,
-            pipeline="bad_name",
-            pipeline_params={"param1": 2, "param2": "value"},  # wrong param1
-        )
-    assert str(exc.value) == "Experiment 'test' has unknown pipeline 'bad_name'"
-
-    # Bad param name and type
-    with pytest.raises(ValueError) as exc:
-        Experiment(
-            **misc,  # type: ignore
-            sequence_length=1000,
-            pipeline="fake",
-            pipeline_params={"bad_param": 2, "param2": "value"},  # wrong param1
-        )
-    assert str(exc.value) == "Invalid parameter 'bad_param' for pipeline 'fake'"
-
-    # Bad param type
-    with pytest.raises(ValueError) as exc:
-        Experiment(
-            **misc,  # type: ignore
-            sequence_length=1000,
-            pipeline="fake",
-            pipeline_params={"param1": 2.0, "param2": "value"},  # wrong param1
-        )
-    assert (
-        str(exc.value)
-        == "Invalid type for parameter 'param1' for pipeline 'fake': expected 'int', got 'float'"
-    )
-
-    # Too long length
-    Experiment(
-        **misc,  # type: ignore
-        sequence_length=2000000,
-        pipeline="fake",
-        pipeline_params={"param1": 1, "param2": "value"},
-    )
-
-    captured = capsys.readouterr()
-    assert (
-        captured.out
-        == "Warning: Experiment 'test' has sequence_length 2000000 > dataset length 15302, reducing to 15302\n"
-    )
diff --git a/tests/test_io.py b/tests/test_io.py
new file mode 100644
index 00000000..93937ada
--- /dev/null
+++ b/tests/test_io.py
@@ -0,0 +1,79 @@
+from pathlib import Path
+from evalio import types as ty
+import numpy as np
+
+
+def make_exp() -> ty.Experiment:
+    return ty.Experiment(
+        name="test",
+        status=ty.ExperimentStatus.Complete,
+        sequence="newer_college_2020/short_experiment",
+        sequence_length=1000,
+        pipeline="fake",
+        pipeline_version="0.1.0",
+        pipeline_params={"param1": 1, "param2": "value"},
+        total_elapsed=10.5,
+        max_step_elapsed=0.24,
+    )
+
+
+def test_metadata_serde():
+    exp = make_exp()
+    out = ty.Metadata.from_yaml(exp.to_yaml())
+    assert exp == out
+
+    gt = ty.GroundTruth(sequence="newer_college_2020/short_experiment")
+    out = ty.Metadata.from_yaml(gt.to_yaml())
+    assert gt == out
+
+
+def test_trajectory_serde(tmp_path: Path):
+    path = tmp_path / "traj.csv"
+
+    traj = ty.Trajectory(
+        stamps=[ty.Stamp.from_sec(i) for i in range(5)],
+        poses=[ty.SE3.exp(np.random.rand(6)) for _ in range(5)],
+        metadata=make_exp(),
+    )
+
+    if traj.metadata is not None:
+        traj.metadata.file = path
+
+    traj.to_file(path)
+
+    loaded = ty.Trajectory.from_file(path)
+    assert isinstance(loaded, ty.Trajectory)
+    assert traj.stamps == loaded.stamps
+    assert traj.poses == loaded.poses
+    assert traj.metadata == loaded.metadata
+
+
+def test_trajectory_incremental_serde(tmp_path: Path):
+    path = tmp_path / "traj.csv"
+
+    traj = ty.Trajectory(
+        stamps=[ty.Stamp.from_sec(i) for i in range(5)],
+        poses=[ty.SE3.exp(np.random.rand(6)) for _ in range(5)],
+        metadata=make_exp(),
+    )
+
+    if traj.metadata is not None:
+        traj.metadata.file = path
+
+    # poses are automatically written as they are added
+    traj.open(path)
+    traj.append(ty.Stamp.from_sec(5), ty.SE3.exp(np.random.rand(6)))
+    traj.close()
+
+    new_traj = ty.Trajectory.from_file(path)
+    assert traj == new_traj
+
+    # must trigger entire rewrite to update metadata
+    traj.open(path)
+    if traj.metadata is not None:
+        traj.metadata.sequence = "random_name"  # type: ignore
+    traj.rewrite()
+    traj.close()
+
+    new_traj = ty.Trajectory.from_file(path)
+    assert traj == new_traj

From 419194d2955460d9ec75ec30e96aea21309e07cd Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Tue, 30 Sep 2025 14:29:45 -0400
Subject: [PATCH 07/40] Officially moved over to new experiment class

---
 python/evalio/cli/completions.py     |  13 +-
 python/evalio/cli/dataset_manager.py |  97 ++++----
 python/evalio/cli/ls.py              |  10 +-
 python/evalio/cli/parser.py          | 283 ----------------------
 python/evalio/cli/run.py             | 347 ++++++++++++++++++++-------
 python/evalio/cli/writer.py          | 113 ---------
 python/evalio/datasets/__init__.py   |   2 +
 python/evalio/pipelines/parser.py    |  29 ++-
 python/evalio/rerun.py               |  20 +-
 python/evalio/types/base.py          |  28 ++-
 python/evalio/types/extended.py      |  29 ++-
 tests/get_lens.py                    |   4 +-
 tests/make_test_data.py              |   4 +-
 tests/test_cli_ls.py                 |  21 --
 tests/test_csv_loading.py            |   9 +-
 tests/test_dataset_impl.py           |   9 +-
 tests/test_parsing.py                |  27 ++-
 17 files changed, 421 insertions(+), 624 deletions(-)
 delete mode 100644 python/evalio/cli/parser.py
 delete mode 100644 python/evalio/cli/writer.py

diff --git a/python/evalio/cli/completions.py b/python/evalio/cli/completions.py
index 843f7439..f1d83bff 100644
--- a/python/evalio/cli/completions.py
+++ b/python/evalio/cli/completions.py
@@ -1,20 +1,13 @@
-import itertools
 from typing import Annotated, Optional, TypeAlias
 
 import typer
 from rapidfuzz.process import extractOne
 from rich.console import Console
-
-from .parser import DatasetBuilder, PipelineBuilder
+from evalio import datasets as ds, pipelines as pl
 
 err_console = Console(stderr=True)
 
-all_sequences_names = list(
-    itertools.chain.from_iterable(
-        [seq.full_name for seq in d.sequences()] + [f"{d.dataset_name()}/*"]
-        for d in DatasetBuilder.all_datasets().values()
-    )
-)
+all_sequences_names = list(ds.all_sequences().keys())
 
 
 # ------------------------- Completions ------------------------- #
@@ -48,7 +41,7 @@ def validate_datasets(datasets: list[str]) -> list[str]:
     return datasets
 
 
-valid_pipelines = list(PipelineBuilder.all_pipelines().keys())
+valid_pipelines = list(pl.all_pipelines().keys())
 
 
 def complete_pipeline(incomplete: str, ctx: typer.Context):
diff --git a/python/evalio/cli/dataset_manager.py b/python/evalio/cli/dataset_manager.py
index 4def9568..e3541014 100644
--- a/python/evalio/cli/dataset_manager.py
+++ b/python/evalio/cli/dataset_manager.py
@@ -21,30 +21,43 @@
 )
 from rosbags.typesys import Stores, get_typestore
 
-from evalio.datasets import RosbagIter
+import evalio.datasets as ds
 from evalio.utils import print_warning
 
 from .completions import DatasetArg
-from .parser import DatasetBuilder
 
 app = typer.Typer()
 
 
+def parse_datasets(
+    datasets: DatasetArg,
+) -> list[ds.Dataset]:
+    """
+    Parse datasets from command line argument
+    """
+    # parse all datasets
+    valid_datasets = ds.parse_config(datasets)
+    if isinstance(valid_datasets, ds.DatasetConfigError):
+        print_warning(f"Error parsing datasets: {valid_datasets}")
+        return []
+    return [b[0] for b in valid_datasets]
+
+
 @app.command(no_args_is_help=True)
 def dl(datasets: DatasetArg) -> None:
     """
     Download datasets
     """
     # parse all datasets
-    valid_datasets = DatasetBuilder.parse(datasets)
+    valid_datasets = parse_datasets(datasets)
 
     # Check if already downloaded
-    to_download: list[DatasetBuilder] = []
-    for builder in valid_datasets:
-        if builder.is_downloaded():
-            print(f"Skipping download for {builder}, already exists")
+    to_download: list[ds.Dataset] = []
+    for dataset in valid_datasets:
+        if dataset.is_downloaded():
+            print(f"Skipping download for {dataset}, already exists")
         else:
-            to_download.append(builder)
+            to_download.append(dataset)
 
     if len(to_download) == 0:
         print("Nothing to download, finishing")
@@ -52,17 +65,17 @@ def dl(datasets: DatasetArg) -> None:
 
     # download each dataset
     print("Will download: ")
-    for builder in to_download:
-        print(f"  {builder}")
+    for dataset in to_download:
+        print(f"  {dataset}")
     print()
 
-    for builder in to_download:
-        print(f"---------- Beginning {builder} ----------")
+    for dataset in to_download:
+        print(f"---------- Beginning {dataset} ----------")
         try:
-            builder.download()
+            dataset.download()
         except Exception as e:
-            print(f"Error downloading {builder}\n: {e}")
-        print(f"---------- Finished {builder} ----------")
+            print(f"Error downloading {dataset}\n: {e}")
+        print(f"---------- Finished {dataset} ----------")
 
 
 @app.command(no_args_is_help=True)
@@ -84,26 +97,26 @@ def rm(
     If --force is not used, will ask for confirmation.
     """
     # parse all datasets
-    to_remove = DatasetBuilder.parse(datasets)
+    to_remove = parse_datasets(datasets)
 
     print("Will remove: ")
-    for builder in to_remove:
-        print(f"  {builder}")
+    for dataset in to_remove:
+        print(f"  {dataset}")
     print()
 
-    for builder in to_remove:
-        print(f"---------- Beginning {builder} ----------")
+    for dataset in to_remove:
+        print(f"---------- Beginning {dataset} ----------")
         try:
-            print(f"Removing from {builder.dataset.folder}")
-            for f in builder.dataset.files():
+            print(f"Removing from {dataset.folder}")
+            for f in dataset.files():
                 print(f"  Removing {f}")
-                if (builder.dataset.folder / f).is_file():
-                    (builder.dataset.folder / f).unlink()
+                if (dataset.folder / f).is_file():
+                    (dataset.folder / f).unlink()
                 else:
-                    shutil.rmtree(builder.dataset.folder / f, ignore_errors=True)
+                    shutil.rmtree(dataset.folder / f, ignore_errors=True)
         except Exception as e:
-            print(f"Error removing {builder}\n: {e}")
-        print(f"---------- Finished {builder} ----------")
+            print(f"Error removing {dataset}\n: {e}")
+        print(f"---------- Finished {dataset} ----------")
 
 
 def filter_ros1(bag: Path, topics: list[str]) -> None:
@@ -215,27 +228,27 @@ def filter(
     Filter rosbag dataset(s) to only include lidar and imu data. Useful for shrinking disk size.
     """
     # parse all datasets
-    valid_datasets = DatasetBuilder.parse(datasets)
+    valid_datasets = parse_datasets(datasets)
 
     # Check if already downloaded
-    to_filter: list[DatasetBuilder] = []
-    for builder in valid_datasets:
-        if not builder.is_downloaded():
-            print(f"Skipping filter for {builder}, not downloaded")
+    to_filter: list[ds.Dataset] = []
+    for dataset in valid_datasets:
+        if not dataset.is_downloaded():
+            print(f"Skipping filter for {dataset}, not downloaded")
         else:
-            to_filter.append(builder)
+            to_filter.append(dataset)
 
     print("Will filter: ")
-    for builder in to_filter:
-        print(f"  {builder}")
+    for dataset in to_filter:
+        print(f"  {dataset}")
     print()
 
-    for builder in to_filter:
-        print(f"---------- Filtering {builder} ----------")
+    for dataset in to_filter:
+        print(f"---------- Filtering {dataset} ----------")
         # try:
-        data = builder.build().data_iter()
-        if not isinstance(data, RosbagIter):
-            print(f"{builder} is not a RosbagDataset, skipping filtering")
+        data = dataset.data_iter()
+        if not isinstance(data, ds.RosbagIter):
+            print(f"{dataset} is not a RosbagDataset, skipping filtering")
             continue
 
         is2 = (data.path[0] / "metadata.yaml").exists()
@@ -266,5 +279,5 @@ def filter(
                 filter_ros1(bag, topics)
 
         # except Exception as e:
-        #     print(f"Error filtering {builder}\n: {e}")
-        print(f"---------- Finished {builder} ----------")
+        #     print(f"Error filtering {dataset}\n: {e}")
+        print(f"---------- Finished {dataset} ----------")
diff --git a/python/evalio/cli/ls.py b/python/evalio/cli/ls.py
index 8973f29e..21ab6848 100644
--- a/python/evalio/cli/ls.py
+++ b/python/evalio/cli/ls.py
@@ -7,9 +7,7 @@
 from rich.console import Console
 from rich.table import Table
 
-from evalio.datasets.base import Dataset
-
-from .parser import DatasetBuilder, PipelineBuilder
+from evalio import datasets as ds, pipelines as pl
 
 app = typer.Typer()
 
@@ -25,7 +23,7 @@ def unique(lst: list[T]):
     return list(dict.fromkeys(lst))
 
 
-def extract_len(d: Dataset) -> str:
+def extract_len(d: ds.Dataset) -> str:
     """Get the length of a dataset in a human readable format
 
     Args:
@@ -88,7 +86,7 @@ def ls(
     if kind == Kind.datasets:
         # Search for datasets using rapidfuzz
         # TODO: Make it search through sequences as well?
-        all_datasets = list(DatasetBuilder.all_datasets().values())
+        all_datasets = list(ds.all_datasets().values())
         if search is not None:
             to_include = extract_iter(
                 search, [d.dataset_name() for d in all_datasets], score_cutoff=90
@@ -206,7 +204,7 @@ def ls(
     if kind == Kind.pipelines:
         # Search for pipelines using rapidfuzz
         # TODO: Make it search through parameters as well?
-        all_pipelines = list(PipelineBuilder.all_pipelines().values())
+        all_pipelines = list(pl.all_pipelines().values())
         if search is not None:
             to_include = extract_iter(
                 search, [d.name() for d in all_pipelines], score_cutoff=90
diff --git a/python/evalio/cli/parser.py b/python/evalio/cli/parser.py
deleted file mode 100644
index 39701357..00000000
--- a/python/evalio/cli/parser.py
+++ /dev/null
@@ -1,283 +0,0 @@
-import functools
-import importlib
-import itertools
-import os
-from copy import deepcopy
-from dataclasses import dataclass
-from inspect import isclass
-from pathlib import Path
-from types import ModuleType
-from typing import Any, Optional, Sequence, cast
-
-import yaml
-
-import evalio
-from evalio.types import Param
-from evalio.datasets import Dataset
-from evalio.pipelines import Pipeline
-from evalio.utils import print_warning
-
-
-# ------------------------- Parsing input ------------------------- #
-# TODO: Find a better way to handle lengths here
-# TODO: Make an experiment class to wrap all of this?
-@dataclass
-class DatasetBuilder:
-    dataset: Dataset
-    length: Optional[int] = None
-
-    @staticmethod
-    def _search_module(module: ModuleType) -> dict[str, type[Dataset]]:
-        return dict(
-            (cls.dataset_name(), cls)
-            for cls in module.__dict__.values()
-            if isclass(cls)
-            and issubclass(cls, Dataset)
-            and cls.__name__ != evalio.datasets.Dataset.__name__
-        )
-
-    @staticmethod
-    @functools.cache
-    def all_datasets() -> dict[str, type[Dataset]]:
-        datasets = DatasetBuilder._search_module(evalio.datasets)
-
-        # Parse env variable for more
-        if "EVALIO_CUSTOM" in os.environ:
-            for dataset in os.environ["EVALIO_CUSTOM"].split(","):
-                module: ModuleType = importlib.import_module(dataset)
-                datasets |= DatasetBuilder._search_module(module)
-
-        return datasets
-
-    @classmethod
-    @functools.cache
-    def _get_dataset(cls, name: str) -> type[Dataset]:
-        DatasetType = cls.all_datasets().get(name, None)
-        if DatasetType is None:
-            raise ValueError(f"Dataset {name} not found")
-        return DatasetType
-
-    @classmethod
-    def parse(
-        cls,
-        d: None
-        | str
-        | dict[str, int | float | str]
-        | Sequence[dict[str, int | float | str] | str],
-    ) -> list["DatasetBuilder"]:
-        # If empty just return
-        if d is None:
-            return []
-
-        # If just given a dataset name
-        if isinstance(d, str):
-            name, seq = d.split("/")
-            if seq == "*":
-                return [
-                    DatasetBuilder(cls._get_dataset(name)(seq))
-                    for seq in cls._get_dataset(name).sequences()
-                ]
-            else:
-                return [DatasetBuilder(cls._get_dataset(name)(seq))]
-
-        # If given a dictionary
-        elif isinstance(d, dict):
-            name, seq = cast(str, d.pop("name")).split("/")
-            length = cast(Optional[int], d.pop("length", None))
-            assert len(d) == 0, f"Invalid dataset configuration {d}"
-            if seq == "*":
-                return [
-                    DatasetBuilder(cls._get_dataset(name)(seq), length)
-                    for seq in cls._get_dataset(name).sequences()
-                ]
-            else:
-                return [DatasetBuilder(cls._get_dataset(name)(seq), length)]
-
-        # If given a list, iterate
-        elif isinstance(d, list):
-            results = [DatasetBuilder.parse(x) for x in d]
-            return list(itertools.chain.from_iterable(results))
-
-        else:
-            raise ValueError(f"Invalid dataset configuration {d}")
-
-    def as_dict(self) -> dict[str, Param]:
-        out: dict[str, Param] = {"name": self.dataset.full_name}
-        if self.length is not None:
-            out["length"] = self.length
-
-        return out
-
-    def is_downloaded(self) -> bool:
-        return self.dataset.is_downloaded()
-
-    def download(self) -> None:
-        self.dataset.download()
-
-    def build(self) -> Dataset:
-        return self.dataset
-
-    def __str__(self) -> str:
-        return self.dataset
-
-
-PIPELINE_NAME = Pipeline.__name__
-PIPELINE_METHODS = [m for m in dir(Pipeline) if not m.startswith("_")]
-
-
-@dataclass
-class PipelineBuilder:
-    name: str
-    pipeline: type[Pipeline]
-    params: dict[str, Param]
-
-    def __post_init__(self):
-        # Make sure all parameters are valid
-        all_params = self.pipeline.default_params()
-        for key in self.params.keys():
-            if key not in all_params:
-                raise ValueError(
-                    f"Invalid parameter {key} for pipeline {self.pipeline.name()}"
-                )
-
-        # Save all params to file later
-        all_params.update(self.params)
-        self.params = all_params
-
-    @staticmethod
-    def _is_pipeline(obj: Any) -> bool:
-        # First check the normal way to short circuit
-        if issubclass(obj, Pipeline):
-            return True
-
-        # If Pipeline isn't a parent
-        if not any(parent.__name__ == PIPELINE_NAME for parent in obj.__mro__):
-            return False
-
-        # If it's missing methods
-        for method in PIPELINE_METHODS:
-            if not hasattr(obj, method):
-                return False
-
-        return True
-
-    @staticmethod
-    def _search_module(module: ModuleType) -> dict[str, type[Pipeline]]:
-        return dict(
-            (cls.name(), cls)
-            for cls in module.__dict__.values()
-            if isclass(cls)
-            and PipelineBuilder._is_pipeline(cls)
-            and cls.__name__ != evalio.pipelines.Pipeline.__name__
-        )
-
-    @staticmethod
-    @functools.lru_cache
-    def all_pipelines() -> dict[str, type[Pipeline]]:
-        pipelines = PipelineBuilder._search_module(evalio.pipelines)
-
-        # Parse env variable for more
-        if "EVALIO_CUSTOM" in os.environ:
-            for dataset in os.environ["EVALIO_CUSTOM"].split(","):
-                module = importlib.import_module(dataset)
-                pipelines |= PipelineBuilder._search_module(module)
-
-        return pipelines
-
-    @classmethod
-    @functools.lru_cache
-    def _get_pipeline(cls, name: str) -> type[Pipeline]:
-        PipelineType = cls.all_pipelines().get(name, None)
-        if PipelineType is None:
-            raise ValueError(f"Pipeline {name} not found")
-        return PipelineType
-
-    @classmethod
-    def parse(
-        cls,
-        p: None | str | dict[str, int | float | str] | Sequence[dict[str, Param] | str],
-    ) -> list["PipelineBuilder"]:
-        # If empty just return
-        if p is None:
-            return []
-
-        # If just given a pipeline name
-        if isinstance(p, str):
-            return [PipelineBuilder(p, cls._get_pipeline(p), {})]
-
-        # If given a dictionary
-        elif isinstance(p, dict):
-            kind = p.pop("pipeline")
-            name = cast(str, p.pop("name", kind))
-            kind = cls._get_pipeline(kind)
-            # If the dictionary has a sweep parameter in it
-            if "sweep" in p:
-                sweep = cast(dict[str, list[Param]], p.pop("sweep"))
-                keys, values = zip(*sweep.items())
-                results: list[PipelineBuilder] = []
-                for options in itertools.product(*values):
-                    parsed_name = deepcopy(name)
-                    params = deepcopy(p)
-                    for k, o in zip(keys, options):
-                        params[k] = o
-                        parsed_name += f"__{k}.{o}"
-                    results.append(PipelineBuilder(parsed_name, kind, params))
-                return results
-            else:
-                return [PipelineBuilder(name, kind, p)]
-
-        # If given a list, iterate
-        elif isinstance(p, list):
-            pipes = [PipelineBuilder.parse(x) for x in p]
-            return list(itertools.chain.from_iterable(pipes))
-
-        else:
-            raise ValueError(f"Invalid pipeline configuration {p}")
-
-    def as_dict(self) -> dict[str, Param]:
-        return {"name": self.name, "pipeline": self.pipeline.name(), **self.params}
-
-    def build(self, dataset: Dataset) -> Pipeline:
-        pipe = self.pipeline()
-        # Set user params
-        params = pipe.set_params(self.params)
-        if len(params) > 0:
-            for k, v in params.items():
-                print_warning(
-                    f"Pipeline {self.name} has unused parameters: {k}={v}. "
-                    "Please check your configuration."
-                )
-
-        # Set dataset params
-        pipe.set_imu_params(dataset.imu_params())
-        pipe.set_lidar_params(dataset.lidar_params())
-        pipe.set_imu_T_lidar(dataset.imu_T_lidar())
-        # Initialize pipeline
-        pipe.initialize()
-        return pipe
-
-    def __str__(self):
-        return f"{self.name}"
-
-
-def parse_config(
-    config_file: Optional[Path],
-) -> tuple[list[PipelineBuilder], list[DatasetBuilder], Optional[Path]]:
-    if config_file is None:
-        return ([], [], None)
-
-    with open(config_file, "r") as f:
-        params = yaml.safe_load(f)
-
-    # get output directory
-    out = Path(params["output_dir"]) if "output_dir" in params else None
-
-    # process datasets & make sure they are downloaded by building
-    datasets = DatasetBuilder.parse(params.get("datasets", None))
-    for d in datasets:
-        d.build()
-
-    # process pipelines
-    pipelines = PipelineBuilder.parse(params.get("pipelines", None))
-
-    return pipelines, datasets, out
diff --git a/python/evalio/cli/run.py b/python/evalio/cli/run.py
index 5677a3a3..bdbfaf63 100644
--- a/python/evalio/cli/run.py
+++ b/python/evalio/cli/run.py
@@ -1,25 +1,51 @@
+import multiprocessing
 from pathlib import Path
-from typing import Annotated, Optional
-
-import numpy as np
-import typer
-from rich import print
+from evalio.cli.completions import DatasetOpt, PipelineOpt
+from evalio.types.base import Trajectory
+from evalio.utils import print_warning
 from tqdm.rich import tqdm
+import yaml
 
-from evalio.cli.completions import DatasetOpt, PipelineOpt
+from evalio import datasets as ds, pipelines as pl, types as ty
 from evalio.rerun import RerunVis, VisArgs
-from evalio.types import ImuMeasurement, LidarMeasurement
-from evalio.utils import print_warning
 
-from .parser import DatasetBuilder, PipelineBuilder, parse_config
-from .stats import eval
-from .writer import TrajectoryWriter, save_config, save_gt
+# from .stats import evaluate
+
+from rich import print
+from typing import Optional, Annotated, cast
+import typer
+
+from time import time
+
 
 app = typer.Typer()
 
 
+# def save_config(
+#     pipelines: Sequence[PipelineBuilder],
+#     datasets: Sequence[DatasetBuilder],
+#     output: Path,
+# ):
+#     # If it's just a file, don't save the entire config file
+#     if output.suffix == ".csv":
+#         return
+
+#     print(f"Saving config to {output}")
+
+#     output.mkdir(parents=True, exist_ok=True)
+#     path = output / "config.yaml"
+
+#     out = dict()
+#     out["datasets"] = [d.as_dict() for d in datasets]
+#     out["pipelines"] = [p.as_dict() for p in pipelines]
+
+#     with open(path, "w") as f:
+#         yaml.dump(out, f)
+
+
 @app.command(no_args_is_help=True, name="run", help="Run pipelines on datasets")
 def run_from_cli(
+    # Config file
     config: Annotated[
         Optional[Path],
         typer.Option(
@@ -30,6 +56,7 @@ def run_from_cli(
             show_default=False,
         ),
     ] = None,
+    # Manual options
     in_datasets: DatasetOpt = None,
     in_pipelines: PipelineOpt = None,
     in_out: Annotated[
@@ -42,6 +69,7 @@ def run_from_cli(
             show_default=False,
         ),
     ] = None,
+    # misc options
     length: Annotated[
         Optional[int],
         typer.Option(
@@ -71,25 +99,43 @@ def run_from_cli(
             parser=VisArgs.parse,
         ),
     ] = None,
+    rerun_failed: Annotated[
+        bool,
+        typer.Option(
+            "--rerun-failed",
+            help="Rerun failed experiments. If not set, will skip previously failed experiments.",
+            show_default=False,
+        ),
+    ] = False,
 ):
     if (in_pipelines or in_datasets or length) and config:
         raise typer.BadParameter(
             "Cannot specify both config and manual options", param_hint="run"
         )
 
-    # Go through visualization options
-    if show is None:
-        vis_args = VisArgs(show=visualize)
-    else:
-        vis_args = show
-    vis = RerunVis(vis_args)
-
-    # Parse the config file if provided
+    # ------------------------- Parse Config file ------------------------- #
     if config is not None:
-        pipelines, datasets, out = parse_config(config)
-        if out is None:
-            out = Path("./evalio_results") / config.stem
+        # load from yaml
+        with open(config, "r") as f:
+            params = yaml.safe_load(f)
+
+        if "datasets" not in params:
+            raise typer.BadParameter(
+                "No datasets specified in config", param_hint="run"
+            )
+        if "pipelines" not in params:
+            raise typer.BadParameter(
+                "No pipelines specified in config", param_hint="run"
+            )
+
+        datasets = ds.parse_config(params.get("datasets", None))
+        pipelines = pl.parse_config(params.get("pipelines", None))
 
+        out = (
+            params["output_dir"] if "output_dir" in params else Path("./evalio_results")
+        )
+
+    # ------------------------- Parse manual options ------------------------- #
     else:
         if in_pipelines is None:
             raise typer.BadParameter(
@@ -100,12 +146,15 @@ def run_from_cli(
                 "Must specify at least one dataset", param_hint="run"
             )
 
-        pipelines = PipelineBuilder.parse(in_pipelines)
-        datasets = DatasetBuilder.parse(in_datasets)
+        if length is not None:
+            temp_datasets: list[ds.DatasetConfig] = [
+                {"name": d, "length": length} for d in in_datasets
+            ]
+        else:
+            temp_datasets = [{"name": d} for d in in_datasets]
 
-        if length:
-            for d in datasets:
-                d.length = length
+        pipelines = pl.parse_config(in_pipelines)
+        datasets = ds.parse_config(temp_datasets)
 
         if in_out is None:
             print_warning("Output directory not set. Defaulting to './evalio_results'")
@@ -113,13 +162,61 @@ def run_from_cli(
         else:
             out = in_out
 
+    # ------------------------- Miscellaneous ------------------------- #
+    # error out if either is wrong
+    if isinstance(datasets, ds.DatasetConfigError):
+        raise typer.BadParameter(
+            f"Error in datasets config: {datasets}", param_hint="run"
+        )
+    if isinstance(pipelines, pl.PipelineConfigError):
+        raise typer.BadParameter(
+            f"Error in pipelines config: {pipelines}", param_hint="run"
+        )
+
     if out.suffix == ".csv" and (len(pipelines) > 1 or len(datasets) > 1):
         raise typer.BadParameter(
             "Output must be a directory when running multiple experiments",
             param_hint="run",
         )
 
-    run(pipelines, datasets, out, vis)
+    print(
+        f"Running {plural(len(datasets), 'dataset')} => {plural(len(pipelines) * len(datasets), 'experiment')}"
+    )
+    dtime = sum(le / d.lidar_params().rate for d, le in datasets)
+    dtime *= len(pipelines)
+    if dtime > 3600:
+        print(f"Estimated time (if real-time): {dtime / 3600:.2f} hours")
+    elif dtime > 60:
+        print(f"Estimated time (if real-time): {dtime / 60:.2f} minutes")
+    else:
+        print(f"Estimated time (if real-time): {dtime:.2f} seconds")
+    print(f"Output will be saved to {out}\n")
+
+    # Go through visualization options
+    if show is None:
+        vis_args = VisArgs(show=visualize)
+    else:
+        vis_args = show
+    vis = RerunVis(vis_args, [p[0] for p in pipelines])
+
+    # save_config(pipelines, datasets, out)
+
+    # ------------------------- Convert to experiments ------------------------- #
+    experiments = [
+        ty.Experiment(
+            name=name,
+            sequence=sequence,
+            sequence_length=length,
+            pipeline=pipeline,
+            pipeline_version=pipeline.version(),
+            pipeline_params=params,
+            file=out / f"{name}.csv",
+        )
+        for sequence, length in datasets
+        for name, pipeline, params in pipelines
+    ]
+
+    run(experiments, vis, rerun_failed)
 
 
 def plural(num: int, word: str) -> str:
@@ -127,63 +224,143 @@ def plural(num: int, word: str) -> str:
 
 
 def run(
-    pipelines: list[PipelineBuilder],
-    datasets: list[DatasetBuilder],
-    output: Path,
+    experiments: list[ty.Experiment],
     vis: RerunVis,
+    rerun_failed: bool,
 ):
-    print(
-        f"Running {plural(len(pipelines), 'pipeline')} on {plural(len(datasets), 'dataset')} => {plural(len(pipelines) * len(datasets), 'experiment')}"
-    )
-    lengths = [
-        min(d.length if d.length is not None else np.inf, len(d.build()))
-        for d in datasets
+    # Make sure everything is in the experiments that we need
+    len_before = len(experiments)
+    experiments = [
+        exp
+        for exp in experiments
+        if isinstance(exp.sequence, ds.Dataset)
+        and not isinstance(exp.pipeline, str)
+        and exp.file is not None
     ]
-    dtime = sum(le / d.dataset.lidar_params().rate for le, d in zip(lengths, datasets))  # type: ignore
-    dtime *= len(pipelines)
-    if dtime > 3600:
-        print(f"Estimated time (if real-time): {dtime / 3600:.2f} hours")
-    elif dtime > 60:
-        print(f"Estimated time (if real-time): {dtime / 60:.2f} minutes")
-    else:
-        print(f"Estimated time (if real-time): {dtime:.2f} seconds")
-    print(f"Output will be saved to {output}\n")
-    save_config(pipelines, datasets, output)
-
-    for dbuilder in datasets:
-        save_gt(output, dbuilder)
-        vis.new_recording(dbuilder.build(), pipelines)
-
-        # Found how much we'll be iterating
-        length = len(dbuilder.build().data_iter())
-        if dbuilder.length is not None and dbuilder.length < length:
-            length = dbuilder.length
-
-        for pbuilder in pipelines:
-            print(f"Running {pbuilder} on {dbuilder}")
-            # Build everything
-            dataset = dbuilder.build()
-            pipe = pbuilder.build(dataset)
-            writer = TrajectoryWriter(output, pbuilder, dbuilder)
-            vis.new_pipe(pbuilder.name)
-
-            # Run the pipeline
-            loop = tqdm(total=length)
-            for data in dbuilder.build():
-                if isinstance(data, ImuMeasurement):
-                    pipe.add_imu(data)
-                elif isinstance(data, LidarMeasurement):  # pyright: ignore reportUnnecessaryIsInstance
-                    features = pipe.add_lidar(data)
-                    pose = pipe.pose()
-                    writer.write(data.stamp, pose)
-
-                    vis.log(data, features, pose, pipe)
-
-                    loop.update()
-                    if loop.n >= length:
-                        loop.close()
-                        break
-
-            writer.close()
-
-    eval([str(output)], False, "RTEt")
+    if len(experiments) < len_before:
+        print_warning(
+            f"Some experiments were invalid and will be skipped ({len_before - len(experiments)} out of {len_before})"
+        )
+
+    prev_dataset = None
+    for exp in experiments:
+        # For the type checker
+        if (
+            isinstance(exp.sequence, str)  # type: ignore
+            or isinstance(exp.pipeline, str)
+            or exp.file is None
+        ):
+            continue
+
+        # save ground truth if we haven't already
+        if not (gt_file := exp.file.parent / "gt.csv").exists():
+            exp.sequence.ground_truth().to_file(gt_file)
+
+        # start vis if needed
+        if prev_dataset != exp.sequence:
+            prev_dataset = exp.sequence
+            vis.new_dataset(exp.sequence)
+
+        # Figure out the status of the experiment
+        if not exp.file.exists() or exp.file.stat().st_size == 0:
+            status = ty.ExperimentStatus.NotRun
+        else:
+            traj = ty.Trajectory.from_file(exp.file)
+            if isinstance(traj, ty.Trajectory) and isinstance(
+                traj.metadata, ty.Experiment
+            ):
+                status = traj.metadata.status
+            else:
+                status = ty.ExperimentStatus.Fail
+
+        # Do something based on the status
+        info = f"{exp.pipeline.name()} on {exp.sequence}"
+        match status:
+            case ty.ExperimentStatus.Complete:
+                print(f"Skipping {info}, already finished")
+                continue
+            case ty.ExperimentStatus.Fail:
+                if rerun_failed:
+                    print(f"Rerunning {info}, previously failed")
+                else:
+                    print(f"Skipping {info}, previously failed")
+                    continue
+            case ty.ExperimentStatus.Started:
+                print(f"Overwriting {info}")
+            case ty.ExperimentStatus.NotRun:
+                print(f"Running {info}")
+
+        # Run the pipeline in a different process so we can recover from segfaults
+        process = multiprocessing.Process(target=run_single, args=(exp, vis))
+        process.start()
+        process.join()
+        exitcode = process.exitcode
+        process.close()
+
+        # If it failed, mark the status as failed
+        if exitcode != 0:
+            exp.status = ty.ExperimentStatus.Fail
+            traj = ty.Trajectory.from_file(exp.file)
+            if isinstance(traj, ty.Trajectory) and isinstance(
+                traj.metadata, ty.Experiment
+            ):
+                traj.metadata = exp
+                traj.to_file()
+            else:
+                Trajectory(metadata=exp).to_file()
+
+    # if len(experiments) > 1:
+    #     if (file := experiments[0].file) is not None:
+    #         evaluate([str(file.parent)])
+
+
+def run_single(
+    exp: ty.Experiment,
+    vis: RerunVis,
+):
+    # Build everything
+    output = exp.setup()
+    if isinstance(output, (ds.DatasetConfigError, pl.PipelineConfigError)):
+        print_warning(f"Error setting up experiment {exp.name}: {output}")
+        return
+    pipe, dataset = cast(tuple[pl.Pipeline, ds.Dataset], output)
+    traj = ty.Trajectory(metadata=exp)
+    traj.open(exp.file)
+    vis.new_pipe(exp.name)
+
+    time_running = 0.0
+    time_max = 0.0
+    time_total = 0.0
+
+    loop = tqdm(total=exp.sequence_length)
+    for data in dataset:
+        if isinstance(data, ty.ImuMeasurement):
+            start = time()
+            pipe.add_imu(data)
+            time_running += time() - start
+        elif isinstance(data, ty.LidarMeasurement):  # type: ignore
+            start = time()
+            features = pipe.add_lidar(data)
+            pose = pipe.pose()
+            time_running += time() - start
+
+            time_total += time_running
+            if time_running > time_max:
+                time_max = time_running
+            time_running = 0.0
+
+            traj.append(data.stamp, pose)
+            vis.log(data, features, pose, pipe)
+
+            loop.update()
+            if loop.n >= exp.sequence_length:
+                loop.close()
+                break
+
+    loop.close()
+    if isinstance(traj.metadata, ty.Experiment):
+        traj.metadata.status = ty.ExperimentStatus.Complete
+        traj.metadata.total_elapsed = time_total
+        traj.metadata.max_step_elapsed = time_max
+    traj.rewrite()
+    traj.close()
diff --git a/python/evalio/cli/writer.py b/python/evalio/cli/writer.py
deleted file mode 100644
index b6975ee6..00000000
--- a/python/evalio/cli/writer.py
+++ /dev/null
@@ -1,113 +0,0 @@
-import atexit
-import csv
-from pathlib import Path
-from typing import Sequence
-
-import yaml
-
-from evalio import Param
-from evalio.types import SE3, Stamp
-
-from .parser import DatasetBuilder, PipelineBuilder
-
-
-def save_config(
-    pipelines: Sequence[PipelineBuilder],
-    datasets: Sequence[DatasetBuilder],
-    output: Path,
-):
-    # If it's just a file, don't save the entire config file
-    if output.suffix == ".csv":
-        return
-
-    print(f"Saving config to {output}")
-
-    output.mkdir(parents=True, exist_ok=True)
-    path = output / "config.yaml"
-
-    out: dict[str, list[dict[str, Param]]] = dict()
-    out["datasets"] = [d.as_dict() for d in datasets]
-    out["pipelines"] = [p.as_dict() for p in pipelines]
-
-    with open(path, "w") as f:
-        yaml.dump(out, f)
-
-
-class TrajectoryWriter:
-    def __init__(self, path: Path, pipeline: PipelineBuilder, dataset: DatasetBuilder):
-        if path.suffix != ".csv":
-            path = path / dataset.dataset.full_name
-            path.mkdir(parents=True, exist_ok=True)
-            path /= f"{pipeline.name}.csv"
-
-        # write metadata to the header
-        # TODO: Could probably automate this using pyserde somehow
-        self.path = path
-        self.file = open(path, "w")
-        self.file.write(f"# name: {pipeline.name}\n")
-        self.file.write(f"# pipeline: {pipeline.pipeline.name()}\n")
-        self.file.write(f"# version: {pipeline.pipeline.version()}\n")
-        for key, value in pipeline.params.items():
-            self.file.write(f"# {key}: {value}\n")
-        self.file.write("#\n")
-        self.file.write(f"# dataset: {dataset.dataset.dataset_name()}\n")
-        self.file.write(f"# sequence: {dataset.dataset.seq_name}\n")
-        if dataset.length is not None:
-            self.file.write(f"# length: {dataset.length}\n")
-        self.file.write("#\n")
-        self.file.write("# timestamp, x, y, z, qx, qy, qz, qw\n")
-
-        self.writer = csv.writer(self.file)
-
-        self.index = 0
-
-        atexit.register(self.close)
-
-    def write(self, stamp: Stamp, pose: SE3):
-        self.writer.writerow(
-            [
-                f"{stamp.sec}.{stamp.nsec:09}",
-                pose.trans[0],
-                pose.trans[1],
-                pose.trans[2],
-                pose.rot.qx,
-                pose.rot.qy,
-                pose.rot.qz,
-                pose.rot.qw,
-            ]
-        )
-        # print(f"Wrote {self.index}")
-        self.index += 1
-
-    def close(self):
-        self.file.close()
-
-
-def save_gt(output: Path, dataset: DatasetBuilder):
-    if output.suffix == ".csv":
-        return
-
-    gt = dataset.build().ground_truth()
-    path = output / dataset.dataset.full_name
-    path.mkdir(parents=True, exist_ok=True)
-    path = path / "gt.csv"
-    with open(path, "w") as f:
-        f.write(f"# dataset: {dataset.dataset.dataset_name()}\n")
-        f.write(f"# sequence: {dataset.dataset.seq_name}\n")
-        f.write("# gt: True\n")
-        f.write("#\n")
-        f.write("# timestamp, x, y, z, qx, qy, qz, qw\n")
-        writer = csv.writer(f)
-        for stamp, pose in gt:
-            writer.writerow(
-                [
-                    stamp.to_sec(),
-                    pose.trans[0],
-                    pose.trans[1],
-                    pose.trans[2],
-                    pose.rot.qx,
-                    pose.rot.qy,
-                    pose.rot.qz,
-                    pose.rot.qw,
-                ]
-            )
diff --git a/python/evalio/datasets/__init__.py b/python/evalio/datasets/__init__.py
index a1f5a658..5e762e88 100644
--- a/python/evalio/datasets/__init__.py
+++ b/python/evalio/datasets/__init__.py
@@ -21,6 +21,7 @@
     SequenceNotFound,
     InvalidDatasetConfig,
     DatasetConfigError,
+    DatasetConfig,
 )
 
 __all__ = [
@@ -49,4 +50,5 @@
     "SequenceNotFound",
     "InvalidDatasetConfig",
     "DatasetConfigError",
+    "DatasetConfig",
 ]
diff --git a/python/evalio/pipelines/parser.py b/python/evalio/pipelines/parser.py
index bec54421..33855965 100644
--- a/python/evalio/pipelines/parser.py
+++ b/python/evalio/pipelines/parser.py
@@ -123,10 +123,11 @@ def get_pipeline(name: str) -> type[Pipeline] | PipelineNotFound:
 def _sweep(
     sweep: dict[str, Param],
     params: dict[str, Param],
+    name: str,
     pipe: type[Pipeline],
-) -> list[tuple[type[Pipeline], dict[str, Param]]] | PipelineConfigError:
+) -> list[tuple[str, type[Pipeline], dict[str, Param]]] | PipelineConfigError:
     keys, values = zip(*sweep.items())
-    results: list[tuple[type[Pipeline], dict[str, Param]]] = []
+    results: list[tuple[str, type[Pipeline], dict[str, Param]]] = []
     for options in itertools.product(*values):
         p = params.copy()
         for k, o in zip(keys, options):
@@ -134,7 +135,7 @@ def _sweep(
         err = validate_params(pipe, p)
         if err is not None:
             return err
-        results.append((pipe, p))
+        results.append((name, pipe, p))
     return results
 
 
@@ -166,7 +167,7 @@ def validate_params(
 
 def parse_config(
     p: str | dict[str, Param] | Sequence[str | dict[str, Param]],
-) -> list[tuple[type[Pipeline], dict[str, Param]]] | PipelineConfigError:
+) -> list[tuple[str, type[Pipeline], dict[str, Param]]] | PipelineConfigError:
     """Parse a pipeline configuration.
 
     Args:
@@ -179,33 +180,41 @@ def parse_config(
         pipe = get_pipeline(p)
         if isinstance(pipe, PipelineNotFound):
             return pipe
-        return [(pipe, {})]
+        return [(p, pipe, {})]
 
     elif isinstance(p, dict):
+        pipe_name = p.pop("pipeline", None)
+        if pipe_name is None:
+            return InvalidPipelineConfig(f"Need pipeline name: {str(p)}")
+        pipe_name = cast(str, pipe_name)
+
         name = p.pop("name", None)
         if name is None:
-            return InvalidPipelineConfig(f"Need pipeline name: {str(p)}")
+            name = pipe_name
+        name = cast(str, name)
 
-        pipe = get_pipeline(cast(str, name))
+        pipe = get_pipeline(pipe_name)
         if isinstance(pipe, PipelineNotFound):
             return pipe
 
         if "sweep" in p:
             sweep = cast(dict[str, Param], p.pop("sweep"))
-            return _sweep(sweep, p, pipe)
+            return _sweep(sweep, p, name, pipe)
         else:
             err = validate_params(pipe, p)
             if err is not None:
                 return err
 
-            return [(pipe, p)]
+            return [(name, pipe, p)]
 
     elif isinstance(p, list):
         results = [parse_config(x) for x in p]
         for r in results:
             if isinstance(r, PipelineConfigError):
                 return r
-        results = cast(list[list[tuple[type[Pipeline], dict[str, Param]]]], results)
+        results = cast(
+            list[list[tuple[str, type[Pipeline], dict[str, Param]]]], results
+        )
         return list(itertools.chain.from_iterable(results))
 
     else:
diff --git a/python/evalio/rerun.py b/python/evalio/rerun.py
index eee577cf..59f29796 100644
--- a/python/evalio/rerun.py
+++ b/python/evalio/rerun.py
@@ -7,7 +7,6 @@
 import typer
 from numpy.typing import NDArray
 
-from evalio.cli.parser import PipelineBuilder
 from evalio.datasets import Dataset
 from evalio.pipelines import Pipeline
 from evalio.stats import _check_overstep
@@ -74,12 +73,13 @@ def parse(opts: str) -> "VisArgs":
     )
 
     class RerunVis:  # type: ignore
-        def __init__(self, args: VisArgs):
+        def __init__(self, args: VisArgs, pipeline_names: list[str]):
             self.args = args
 
             # To be set during new_recording
             self.lidar_params: Optional[LidarParams] = None
             self.gt: Optional[Trajectory] = None
+            self.pipeline_names = pipeline_names
 
             # To be found during log
             self.gt_o_T_imu_o: Optional[SE3] = None
@@ -108,15 +108,13 @@ def __init__(self, args: VisArgs):
                 + [skybox_light_rgb(dir) for dir in directions]
             )
 
-        def _blueprint(self, pipelines: list[PipelineBuilder]) -> rr.BlueprintLike:
+        def _blueprint(self) -> rr.BlueprintLike:
             # Eventually we'll be able to glob these, but for now, just take in the names beforehand
             # https://github.com/rerun-io/rerun/issues/6673
             # Once this is closed, we'll be able to remove pipelines as a parameter here and in new_recording
             overrides: OverrideType = {
-                f"{p.name}/imu": [
-                    rrb.VisualizerOverrides(rrb.visualizers.Transform3DArrows)
-                ]
-                for p in pipelines
+                f"{n}/imu": [rrb.VisualizerOverrides(rrb.visualizers.Transform3DArrows)]
+                for n in self.pipeline_names
             }
 
             if self.args.image:
@@ -130,7 +128,7 @@ def _blueprint(self, pipelines: list[PipelineBuilder]) -> rr.BlueprintLike:
             else:
                 return rrb.Blueprint(rrb.Spatial3DView(overrides=overrides))
 
-        def new_recording(self, dataset: Dataset, pipelines: list[PipelineBuilder]):
+        def new_dataset(self, dataset: Dataset):
             if not self.args.show:
                 return
 
@@ -141,7 +139,7 @@ def new_recording(self, dataset: Dataset, pipelines: list[PipelineBuilder]):
             }
             self.rec = rr.RecordingStream(**self.recording_params)
             self.rec.connect_grpc()
-            self.rec.send_blueprint(self._blueprint(pipelines))
+            self.rec.send_blueprint(self._blueprint())
 
             self.gt = dataset.ground_truth()
             self.lidar_params = dataset.lidar_params()
@@ -453,11 +451,11 @@ def convert(
 except Exception:
 
     class RerunVis:
-        def __init__(self, args: VisArgs) -> None:
+        def __init__(self, args: VisArgs, pipeline_names: list[str]) -> None:
             if args.show:
                 print_warning("Rerun not found, visualization disabled")
 
-        def new_recording(self, dataset: Dataset, pipelines: list[PipelineBuilder]):
+        def new_dataset(self, dataset: Dataset):
             pass
 
         def log(
diff --git a/python/evalio/types/base.py b/python/evalio/types/base.py
index abfbf3af..af6352e2 100644
--- a/python/evalio/types/base.py
+++ b/python/evalio/types/base.py
@@ -6,7 +6,7 @@
 
 from __future__ import annotations
 
-from dataclasses import asdict, dataclass
+from dataclasses import asdict, dataclass, field
 import csv
 from _csv import Writer
 from enum import Enum
@@ -95,9 +95,9 @@ class GroundTruth(Metadata):
 
 @dataclass(kw_only=True)
 class Trajectory:
-    stamps: list[Stamp]
+    stamps: list[Stamp] = field(default_factory=list)
     """List of timestamps for each pose."""
-    poses: list[SE3]
+    poses: list[SE3] = field(default_factory=list)
     """List of poses, in the same order as the timestamps."""
     metadata: Optional[Metadata] = None
     """Metadata associated with the trajectory, such as the dataset name or other information."""
@@ -276,9 +276,23 @@ def _write(self):
         self._file.write("# timestamp, x, y, z, qx, qy, qz, qw\n")
         self._csv_writer.writerows(self._serialize_pose(s, p) for s, p in self)
 
-    def open(self, path: Path):
-        if self.metadata is not None:
-            self.metadata.file = path
+    def open(self, path: Optional[Path] = None):
+        """Open a CSV file for writing.
+
+        This will overwrite any existing file. If no path is provided, will use the path in the metadata, if it exists.
+
+        Args:
+            path (Optional[Path], optional): Path to the CSV file. Defaults to None.
+        """
+        if path is not None:
+            pass
+        elif self.metadata is not None and self.metadata.file is not None:
+            path = self.metadata.file
+        else:
+            print_warning(
+                "Trajectory.open: No metadata or path provided, cannot set metadata file."
+            )
+            return
         self._file = path.open("w")
         self._csv_writer = csv.writer(self._file)
         self._write()
@@ -292,7 +306,7 @@ def close(self):
         else:
             print_warning("Trajectory.close: No file to close.")
 
-    def to_file(self, path: Path):
+    def to_file(self, path: Optional[Path] = None):
         self.open(path)
         self.close()
 
diff --git a/python/evalio/types/extended.py b/python/evalio/types/extended.py
index 7cecea45..bb8e2027 100644
--- a/python/evalio/types/extended.py
+++ b/python/evalio/types/extended.py
@@ -17,23 +17,24 @@ class ExperimentStatus(Enum):
     Complete = "complete"
     Fail = "fail"
     Started = "started"
+    NotRun = "not_run"
 
 
 @dataclass(kw_only=True)
 class Experiment(Metadata):
     name: str
     """Name of the experiment."""
-    sequence: str
+    sequence: str | ds.Dataset
     """Dataset used to run the experiment."""
     sequence_length: int
     """Length of the sequence, if set"""
-    pipeline: str
+    pipeline: str | type[pl.Pipeline]
     """Pipeline used to generate the trajectory."""
     pipeline_version: str
     """Version of the pipeline used."""
     pipeline_params: dict[str, Param]
     """Parameters used for the pipeline."""
-    status: ExperimentStatus = ExperimentStatus.Started
+    status: ExperimentStatus = ExperimentStatus.NotRun
     """Status of the experiment, e.g. "success", "failure", etc."""
     total_elapsed: Optional[float] = None
     """Total time taken for the experiment, as a string."""
@@ -43,6 +44,11 @@ class Experiment(Metadata):
     def to_dict(self) -> dict[str, Any]:
         d = super().to_dict()
         d["status"] = self.status.value
+        if isinstance(self.pipeline, type):
+            d["pipeline"] = self.pipeline.name()
+        if isinstance(self.sequence, ds.Dataset):
+            d["sequence"] = self.sequence.full_name
+
         return d
 
     @classmethod
@@ -52,12 +58,17 @@ def from_dict(cls, data: dict[str, Any]) -> Self:
 
         return super().from_dict(data)
 
-    def make_pipeline(
+    def setup(
         self,
-    ) -> pl.Pipeline | ds.DatasetConfigError | pl.PipelineConfigError:
-        ThisPipeline = pl.get_pipeline(self.pipeline)
-        if isinstance(ThisPipeline, pl.PipelineNotFound):
-            return ThisPipeline
+    ) -> (
+        tuple[pl.Pipeline, ds.Dataset] | ds.DatasetConfigError | pl.PipelineConfigError
+    ):
+        if isinstance(self.pipeline, str):
+            ThisPipeline = pl.get_pipeline(self.pipeline)
+            if isinstance(ThisPipeline, pl.PipelineNotFound):
+                return ThisPipeline
+        else:
+            ThisPipeline = self.pipeline
 
         dataset = ds.get_sequence(self.sequence)
         if isinstance(dataset, ds.SequenceNotFound):
@@ -82,4 +93,4 @@ def make_pipeline(
         # Initialize pipeline
         pipe.initialize()
 
-        return pipe
+        return pipe, dataset
diff --git a/tests/get_lens.py b/tests/get_lens.py
index 816c5e1b..7fb04bd7 100644
--- a/tests/get_lens.py
+++ b/tests/get_lens.py
@@ -1,10 +1,10 @@
-from evalio.cli.parser import DatasetBuilder
+import evalio.datasets as ds
 from evalio.utils import print_warning
 from rich import print
 
 # Helper script to get the lengths of all datasets
 # Easier than manually checking each
-for d in DatasetBuilder.all_datasets().values():
+for d in ds.all_datasets().values():
     lengths = {}
 
     for seq in d.sequences():
diff --git a/tests/make_test_data.py b/tests/make_test_data.py
index 783dc1ac..8006c2dc 100644
--- a/tests/make_test_data.py
+++ b/tests/make_test_data.py
@@ -1,9 +1,9 @@
 import pickle  # noqa: F401
 from pathlib import Path
 
-from evalio.cli.parser import DatasetBuilder
+import evalio.datasets as ds
 
-dataset_classes = DatasetBuilder.all_datasets()
+dataset_classes = ds.all_datasets()
 datasets = [
     cls(cls.sequences()[0])
     for cls in dataset_classes.values()
diff --git a/tests/test_cli_ls.py b/tests/test_cli_ls.py
index 336e2da4..161f357e 100644
--- a/tests/test_cli_ls.py
+++ b/tests/test_cli_ls.py
@@ -1,27 +1,6 @@
 from evalio.cli.ls import Kind, ls
-from evalio.cli.parser import DatasetBuilder
 
 
 def test_ls_pipelines():
     ls(Kind.datasets)
     ls(Kind.pipelines)
-
-
-def test_dataset_build():
-    all_datasets, all_names = map(
-        list,
-        zip(
-            *[
-                (s, s.full_name)
-                for d in DatasetBuilder.all_datasets().values()
-                for s in d.sequences()
-            ]
-        ),
-    )
-
-    # Test all datasets
-    out = [d.dataset for d in DatasetBuilder.parse(all_names)]
-    if out != all_datasets:
-        assert len(out) == len(all_datasets), f"Missing datasets in parser {all_names}"
-        for d, name in zip(out, all_names):
-            assert out == d, f"Failed on {name}"
diff --git a/tests/test_csv_loading.py b/tests/test_csv_loading.py
index cc2d5a68..ee0044af 100644
--- a/tests/test_csv_loading.py
+++ b/tests/test_csv_loading.py
@@ -4,8 +4,7 @@
 
 import numpy as np
 import pytest
-from evalio.cli.parser import DatasetBuilder
-from evalio.datasets.base import Dataset
+import evalio.datasets as ds
 from evalio.types import (
     SE3,
     GroundTruth,
@@ -18,7 +17,7 @@
 
 # ------------------------- Loading imu & lidar ------------------------- #
 data_dir = Path("tests/data")
-dataset_classes = DatasetBuilder.all_datasets()
+dataset_classes = ds.all_datasets()
 datasets = [
     cls.sequences()[0]
     for cls in dataset_classes.values()
@@ -31,7 +30,7 @@
 
 
 @pytest.mark.parametrize("dataset", datasets)
-def test_load_imu(dataset: Dataset):
+def test_load_imu(dataset: ds.Dataset):
     imu = dataset.get_one_imu()
     with open(data_dir / f"imu_{dataset.dataset_name()}.pkl", "rb") as f:
         imu_cached: ImuMeasurement = pickle.load(f)
@@ -42,7 +41,7 @@ def test_load_imu(dataset: Dataset):
 
 
 @pytest.mark.parametrize("dataset", datasets)
-def test_load_lidar(dataset: Dataset):
+def test_load_lidar(dataset: ds.Dataset):
     lidar = dataset.get_one_lidar()
     with open(data_dir / f"lidar_{dataset.dataset_name()}.pkl", "rb") as f:
         lidar_cached: LidarMeasurement = pickle.load(f)
diff --git a/tests/test_dataset_impl.py b/tests/test_dataset_impl.py
index 66bd5908..d2bb4afb 100644
--- a/tests/test_dataset_impl.py
+++ b/tests/test_dataset_impl.py
@@ -1,13 +1,12 @@
 import pytest
-from evalio.cli.parser import DatasetBuilder
-from evalio.datasets import Dataset
+from evalio import datasets as ds
 
-datasets = DatasetBuilder.all_datasets().values()
+datasets = ds.all_datasets().values()
 
 
 # Test to ensure all datasets implement the required attributes
 @pytest.mark.parametrize("dataset", datasets)
-def test_impl(dataset: type[Dataset]):
+def test_impl(dataset: type[ds.Dataset]):
     attrs = [
         "data_iter",
         "ground_truth_raw",
@@ -20,6 +19,6 @@ def test_impl(dataset: type[Dataset]):
     ]
 
     for a in attrs:
-        assert getattr(dataset, a) != getattr(Dataset, a), (
+        assert getattr(dataset, a) != getattr(ds.Dataset, a), (
             f"{dataset} should implement {a}"
         )
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index ec9e639c..7cc7e565 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -56,23 +56,24 @@ def default_params() -> dict[str, Param]:
 # fmt: off
 PIPELINES: list[Any] = [
     # good ones
-    ("fake", [(FakePipeline, {})]),
-    ({"name": "fake"}, [(FakePipeline, {})]),
-    ({"name": "fake", "param1": 5}, [(FakePipeline, {"param1": 5})]),
-    (["fake", {"name": "fake", "param1": 3}], [(FakePipeline, {}), (FakePipeline, {"param1": 3})]),
-    ({"name": "fake", "sweep": {"param1": [1, 2, 3]}}, [
-        (FakePipeline, {"param1": 1}),
-        (FakePipeline, {"param1": 2}),
-        (FakePipeline, {"param1": 3}),
+    ("fake", [("fake", FakePipeline, {})]),
+    ({"pipeline": "fake"}, [("fake", FakePipeline, {})]),
+    ({"name": "test", "pipeline": "fake"}, [("test", FakePipeline, {})]),
+    ({"pipeline": "fake", "param1": 5}, [("fake", FakePipeline, {"param1": 5})]),
+    (["fake", {"pipeline": "fake", "param1": 3}], [("fake", FakePipeline, {}), ("fake", FakePipeline, {"param1": 3})]),
+    ({"pipeline": "fake", "sweep": {"param1": [1, 2, 3]}}, [
+        ("fake", FakePipeline, {"param1": 1}),
+        ("fake", FakePipeline, {"param1": 2}),
+        ("fake", FakePipeline, {"param1": 3}),
     ]),
     # bad ones
     ("unknown", pl.PipelineNotFound("unknown")),
-    ({"name": "unknown"}, pl.PipelineNotFound("unknown")),
+    ({"pipeline": "unknown"}, pl.PipelineNotFound("unknown")),
     ({"param1": 5}, pl.InvalidPipelineConfig("Need pipeline name: {'param1': 5}")), # type: ignore
-    ({"name": "fake", "param3": 10}, pl.UnusedPipelineParam("param3", "fake")),
-    ({"name": "fake", "param1": "wrong_type"}, pl.InvalidPipelineParamType("param1", int, str)),
-    ({"name": "fake", "sweep": {"param1": [1.0, 2, 3]}}, pl.InvalidPipelineParamType("param1", int, float)),
-    ({"name": "fake", "sweep": {"param3": [1.0, 2, 3]}}, pl.UnusedPipelineParam("param3", "fake")),
+    ({"pipeline": "fake", "param3": 10}, pl.UnusedPipelineParam("param3", "fake")),
+    ({"pipeline": "fake", "param1": "wrong_type"}, pl.InvalidPipelineParamType("param1", int, str)),
+    ({"pipeline": "fake", "sweep": {"param1": [1.0, 2, 3]}}, pl.InvalidPipelineParamType("param1", int, float)),
+    ({"pipeline": "fake", "sweep": {"param3": [1.0, 2, 3]}}, pl.UnusedPipelineParam("param3", "fake")),
 ]
 # fmt: on
 

From 7148ee8ec63a61c544eaef07f65f53d0c8a9fb7d Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Tue, 30 Sep 2025 15:06:17 -0400
Subject: [PATCH 08/40] Fix a handful of small bugs when playing around with
 things

---
 python/evalio/cli/run.py          | 21 ++++++++++++++-------
 python/evalio/pipelines/parser.py |  5 ++++-
 python/evalio/types/extended.py   | 11 ++++++++---
 tests/test_parsing.py             |  6 +++---
 4 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/python/evalio/cli/run.py b/python/evalio/cli/run.py
index bdbfaf63..09718656 100644
--- a/python/evalio/cli/run.py
+++ b/python/evalio/cli/run.py
@@ -202,6 +202,7 @@ def run_from_cli(
     # save_config(pipelines, datasets, out)
 
     # ------------------------- Convert to experiments ------------------------- #
+    out.mkdir(parents=True, exist_ok=True)
     experiments = [
         ty.Experiment(
             name=name,
@@ -246,7 +247,7 @@ def run(
     for exp in experiments:
         # For the type checker
         if (
-            isinstance(exp.sequence, str)  # type: ignore
+            not isinstance(exp.sequence, ds.Dataset)
             or isinstance(exp.pipeline, str)
             or exp.file is None
         ):
@@ -256,11 +257,6 @@ def run(
         if not (gt_file := exp.file.parent / "gt.csv").exists():
             exp.sequence.ground_truth().to_file(gt_file)
 
-        # start vis if needed
-        if prev_dataset != exp.sequence:
-            prev_dataset = exp.sequence
-            vis.new_dataset(exp.sequence)
-
         # Figure out the status of the experiment
         if not exp.file.exists() or exp.file.stat().st_size == 0:
             status = ty.ExperimentStatus.NotRun
@@ -269,7 +265,12 @@ def run(
             if isinstance(traj, ty.Trajectory) and isinstance(
                 traj.metadata, ty.Experiment
             ):
-                status = traj.metadata.status
+                # If the sequence length has changed, mark as not run
+                if traj.metadata.sequence_length != exp.sequence_length:
+                    status = ty.ExperimentStatus.NotRun
+                else:
+                    status = traj.metadata.status
+
             else:
                 status = ty.ExperimentStatus.Fail
 
@@ -290,6 +291,11 @@ def run(
             case ty.ExperimentStatus.NotRun:
                 print(f"Running {info}")
 
+        # start vis if needed
+        if prev_dataset != exp.sequence:
+            prev_dataset = exp.sequence
+            vis.new_dataset(exp.sequence)
+
         # Run the pipeline in a different process so we can recover from segfaults
         process = multiprocessing.Process(target=run_single, args=(exp, vis))
         process.start()
@@ -324,6 +330,7 @@ def run_single(
         print_warning(f"Error setting up experiment {exp.name}: {output}")
         return
     pipe, dataset = cast(tuple[pl.Pipeline, ds.Dataset], output)
+    exp.status = ty.ExperimentStatus.Started
     traj = ty.Trajectory(metadata=exp)
     traj.open(exp.file)
     vis.new_pipe(exp.name)
diff --git a/python/evalio/pipelines/parser.py b/python/evalio/pipelines/parser.py
index 33855965..af44dd36 100644
--- a/python/evalio/pipelines/parser.py
+++ b/python/evalio/pipelines/parser.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from copy import deepcopy
 import importlib
 from inspect import isclass
 import itertools
@@ -129,13 +130,15 @@ def _sweep(
     keys, values = zip(*sweep.items())
     results: list[tuple[str, type[Pipeline], dict[str, Param]]] = []
     for options in itertools.product(*values):
+        parsed_name = deepcopy(name)
         p = params.copy()
         for k, o in zip(keys, options):
             p[k] = o
+            parsed_name += f"__{k}-{o}"
         err = validate_params(pipe, p)
         if err is not None:
             return err
-        results.append((name, pipe, p))
+        results.append((parsed_name, pipe, p))
     return results
 
 
diff --git a/python/evalio/types/extended.py b/python/evalio/types/extended.py
index bb8e2027..61cb4976 100644
--- a/python/evalio/types/extended.py
+++ b/python/evalio/types/extended.py
@@ -55,6 +55,8 @@ def to_dict(self) -> dict[str, Any]:
     def from_dict(cls, data: dict[str, Any]) -> Self:
         if "status" in data:
             data["status"] = ExperimentStatus(data["status"])
+        else:
+            data["status"] = ExperimentStatus.Started
 
         return super().from_dict(data)
 
@@ -70,9 +72,12 @@ def setup(
         else:
             ThisPipeline = self.pipeline
 
-        dataset = ds.get_sequence(self.sequence)
-        if isinstance(dataset, ds.SequenceNotFound):
-            return dataset
+        if isinstance(self.sequence, ds.Dataset):
+            dataset = self.sequence
+        else:
+            dataset = ds.get_sequence(self.sequence)
+            if isinstance(dataset, ds.SequenceNotFound):
+                return dataset
 
         pipe = ThisPipeline()
 
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index 7cc7e565..cef2cfa0 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -62,9 +62,9 @@ def default_params() -> dict[str, Param]:
     ({"pipeline": "fake", "param1": 5}, [("fake", FakePipeline, {"param1": 5})]),
     (["fake", {"pipeline": "fake", "param1": 3}], [("fake", FakePipeline, {}), ("fake", FakePipeline, {"param1": 3})]),
     ({"pipeline": "fake", "sweep": {"param1": [1, 2, 3]}}, [
-        ("fake", FakePipeline, {"param1": 1}),
-        ("fake", FakePipeline, {"param1": 2}),
-        ("fake", FakePipeline, {"param1": 3}),
+        ("fake__param1-1", FakePipeline, {"param1": 1}),
+        ("fake__param1-2", FakePipeline, {"param1": 2}),
+        ("fake__param1-3", FakePipeline, {"param1": 3}),
     ]),
     # bad ones
     ("unknown", pl.PipelineNotFound("unknown")),

From eeebc53ab86db3e2e0f78e53b2dbc22b1f5ddffc Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Tue, 30 Sep 2025 15:24:49 -0400
Subject: [PATCH 09/40] Add default params to parser

---
 python/evalio/pipelines/parser.py | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/python/evalio/pipelines/parser.py b/python/evalio/pipelines/parser.py
index af44dd36..e92fa6a8 100644
--- a/python/evalio/pipelines/parser.py
+++ b/python/evalio/pipelines/parser.py
@@ -123,9 +123,9 @@ def get_pipeline(name: str) -> type[Pipeline] | PipelineNotFound:
 # ------------------------- Handle yaml parsing ------------------------- #
 def _sweep(
     sweep: dict[str, Param],
-    params: dict[str, Param],
     name: str,
     pipe: type[Pipeline],
+    params: dict[str, Param],
 ) -> list[tuple[str, type[Pipeline], dict[str, Param]]] | PipelineConfigError:
     keys, values = zip(*sweep.items())
     results: list[tuple[str, type[Pipeline], dict[str, Param]]] = []
@@ -183,32 +183,36 @@ def parse_config(
         pipe = get_pipeline(p)
         if isinstance(pipe, PipelineNotFound):
             return pipe
-        return [(p, pipe, {})]
+        return [(p, pipe, pipe.default_params())]
 
     elif isinstance(p, dict):
-        pipe_name = p.pop("pipeline", None)
-        if pipe_name is None:
-            return InvalidPipelineConfig(f"Need pipeline name: {str(p)}")
-        pipe_name = cast(str, pipe_name)
-
-        name = p.pop("name", None)
-        if name is None:
-            name = pipe_name
+        # figure out name of pipeline
+        if "pipeline" not in p:
+            return InvalidPipelineConfig(f"Need pipeline: {str(p)}")
+        pipe_name = cast(str, p["pipeline"])
+
+        # figure out the name
+        name = p.pop("name", pipe_name)
         name = cast(str, name)
 
+        # Construct pipeline
         pipe = get_pipeline(pipe_name)
         if isinstance(pipe, PipelineNotFound):
             return pipe
 
+        # Construct params
+        params = pipe.default_params() | p
+
+        # Handle sweeps
         if "sweep" in p:
             sweep = cast(dict[str, Param], p.pop("sweep"))
-            return _sweep(sweep, p, name, pipe)
+            return _sweep(sweep, name, pipe, params)
         else:
-            err = validate_params(pipe, p)
+            err = validate_params(pipe, params)
             if err is not None:
                 return err
 
-            return [(name, pipe, p)]
+            return [(name, pipe, params)]
 
     elif isinstance(p, list):
         results = [parse_config(x) for x in p]

From ebf4e5d397ce02a16de34ccf7f6a7d804e34749d Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Tue, 30 Sep 2025 21:38:08 -0400
Subject: [PATCH 10/40] Clean up stats command A LOT

---
 pyproject.toml              |   4 +-
 python/evalio/cli/run.py    |  36 ++-
 python/evalio/cli/stats.py  | 423 +++++++++++++++++++++++++-----------
 python/evalio/types/base.py |   7 +-
 uv.lock                     |  52 ++++-
 5 files changed, 364 insertions(+), 158 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index d3b1b625..f98758d2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,10 +7,11 @@ description = "Evaluate Lidar-Inertial Odometry on public datasets"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
-    "argcomplete>=3.3.0",
     "distinctipy>=1.3.4",
     "gdown>=5.2.0",
+    "joblib>=1.5.2",
     "numpy",
+    "polars>=1.33.1",
     "pyyaml>=6.0",
     "rapidfuzz>=3.12.2",
     "rosbags>=0.10.10",
@@ -81,6 +82,7 @@ dev-dependencies = [
     "nanobind>=2.9.2",
     "mike>=2.1.3",
     "basedpyright>=1.31.4",
+    "joblib-stubs>=1.5.2.0.20250831",
 ]
 
 [tool.ruff]
diff --git a/python/evalio/cli/run.py b/python/evalio/cli/run.py
index 09718656..4cecd79e 100644
--- a/python/evalio/cli/run.py
+++ b/python/evalio/cli/run.py
@@ -12,7 +12,7 @@
 # from .stats import evaluate
 
 from rich import print
-from typing import Optional, Annotated, cast
+from typing import Optional, Annotated
 import typer
 
 from time import time
@@ -132,7 +132,9 @@ def run_from_cli(
         pipelines = pl.parse_config(params.get("pipelines", None))
 
         out = (
-            params["output_dir"] if "output_dir" in params else Path("./evalio_results")
+            params["output_dir"]
+            if "output_dir" in params
+            else Path("./evalio_results") / config.stem
         )
 
     # ------------------------- Parse manual options ------------------------- #
@@ -202,7 +204,6 @@ def run_from_cli(
     # save_config(pipelines, datasets, out)
 
     # ------------------------- Convert to experiments ------------------------- #
-    out.mkdir(parents=True, exist_ok=True)
     experiments = [
         ty.Experiment(
             name=name,
@@ -211,7 +212,7 @@ def run_from_cli(
             pipeline=pipeline,
             pipeline_version=pipeline.version(),
             pipeline_params=params,
-            file=out / f"{name}.csv",
+            file=out / sequence / f"{name}.csv",
         )
         for sequence, length in datasets
         for name, pipeline, params in pipelines
@@ -258,21 +259,16 @@ def run(
             exp.sequence.ground_truth().to_file(gt_file)
 
         # Figure out the status of the experiment
-        if not exp.file.exists() or exp.file.stat().st_size == 0:
-            status = ty.ExperimentStatus.NotRun
-        else:
-            traj = ty.Trajectory.from_file(exp.file)
-            if isinstance(traj, ty.Trajectory) and isinstance(
-                traj.metadata, ty.Experiment
-            ):
-                # If the sequence length has changed, mark as not run
-                if traj.metadata.sequence_length != exp.sequence_length:
-                    status = ty.ExperimentStatus.NotRun
-                else:
-                    status = traj.metadata.status
-
+        traj = ty.Trajectory.from_file(exp.file)
+        if isinstance(traj, ty.Trajectory) and isinstance(traj.metadata, ty.Experiment):
+            # If the sequence length has changed, mark as started
+            if traj.metadata.sequence_length != exp.sequence_length:
+                status = ty.ExperimentStatus.Started
             else:
-                status = ty.ExperimentStatus.Fail
+                status = traj.metadata.status
+
+        else:
+            status = ty.ExperimentStatus.NotRun
 
         # Do something based on the status
         info = f"{exp.pipeline.name()} on {exp.sequence}"
@@ -326,10 +322,10 @@ def run_single(
 ):
     # Build everything
     output = exp.setup()
-    if isinstance(output, (ds.DatasetConfigError, pl.PipelineConfigError)):
+    if isinstance(output, ds.DatasetConfigError | pl.PipelineConfigError):
         print_warning(f"Error setting up experiment {exp.name}: {output}")
         return
-    pipe, dataset = cast(tuple[pl.Pipeline, ds.Dataset], output)
+    pipe, dataset = output
     exp.status = ty.ExperimentStatus.Started
     traj = ty.Trajectory(metadata=exp)
     traj.open(exp.file)
diff --git a/python/evalio/cli/stats.py b/python/evalio/cli/stats.py
index 9f1d690a..32035e7e 100644
--- a/python/evalio/cli/stats.py
+++ b/python/evalio/cli/stats.py
@@ -1,65 +1,54 @@
 from pathlib import Path
-from typing import Annotated, Optional, Sequence, cast
+from typing import Annotated, Any, Callable, Optional, cast
 
-import typer
-from rich import box
-from rich.console import Console
-from rich.table import Table
+import polars as pl
+import itertools
 
-from evalio import Param, stats
-from evalio.types import Trajectory
 from evalio.utils import print_warning
+from rich.table import Table
+from rich.console import Console
+from rich import box
 
-app = typer.Typer()
-
-
-def dict_diff(dicts: Sequence[dict[str, Param]]) -> list[str]:
-    """Compute which values are different between a list of dictionaries.
-
-    Assumes each dictionary has the same keys.
-
-    Args:
-        dicts (Sequence[dict]): List of dictionaries to compare.
+from evalio import types as ty
+from evalio import stats
 
-    Returns:
-        list[str]: Keys that don't have identical values between all dictionaries.
-    """
+import numpy as np
+import typer
 
-    # quick sanity check
-    size = len(dicts[0])
-    for d in dicts:
-        assert len(d) == size
+import distinctipy
 
-    # compare all dictionaries to find varying keys
-    diff: list[str] = []
-    for k in dicts[0].keys():
-        if any(d[k] != dicts[0][k] for d in dicts):
-            diff.append(k)
+from joblib import Parallel, delayed
 
-    return diff
+app = typer.Typer()
 
 
 def eval_dataset(
     dir: Path,
     visualize: bool,
-    sort: Optional[str],
-    window_size: int,
+    window_kind: stats.WindowKind,
+    window_size: Optional[int | float],
     metric: stats.MetricKind,
     length: Optional[int],
-):
+) -> Optional[list[dict[str, Any]]]:
     # Load all trajectories
-    trajectories: list[Trajectory] = []
+    gt_og: Optional[ty.Trajectory] = None
+    all_trajs: list[ty.Trajectory] = []
     for file_path in dir.glob("*.csv"):
-        traj = Trajectory.from_experiment(file_path)
-        trajectories.append(traj)
-
-    gt_list: list[Trajectory] = []
-    trajs: list[Trajectory] = []
-    for t in trajectories:
-        (gt_list if "gt" in t.metadata else trajs).append(t)
-
-    assert len(gt_list) == 1, f"Found multiple ground truths in {dir}"
-    gt_og = gt_list[0]
+        traj = ty.Trajectory.from_file(file_path)
+        if not isinstance(traj, ty.Trajectory):
+            print_warning(f"Could not load trajectory from {file_path}, skipping.")
+            continue
+        elif isinstance(traj.metadata, ty.GroundTruth):
+            if gt_og is not None:
+                print_warning(f"Multiple ground truths found in {dir}, skipping.")
+                continue
+            gt_og = traj
+        elif isinstance(traj.metadata, ty.Experiment):
+            all_trajs.append(traj)
+
+    if gt_og is None:
+        print_warning(f"No ground truth found in {dir}, skipping.")
+        return None
 
     # Setup visualization
     if visualize:
@@ -71,10 +60,10 @@ def eval_dataset(
 
     rr = None
     convert = None
+    colors = None
     if visualize:
         import rerun as rr
-
-        from evalio.rerun import convert  # type: ignore
+        from evalio.rerun import convert, GT_COLOR
 
         rr.init(
             str(dir),
@@ -83,131 +72,315 @@ def eval_dataset(
         rr.connect_grpc()
         rr.log(
             "gt",
-            convert(gt_og, color=(144, 144, 144)),
+            convert(gt_og, color=GT_COLOR),
             static=True,
         )
 
-    # Group into pipelines so we can compare keys
-    # (other pipelines will have different keys)
-    pipelines = set(cast(str, traj.metadata["pipeline"]) for traj in trajs)
-    grouped_trajs: dict[str, list[Trajectory]] = {p: [] for p in pipelines}
-    for traj in trajs:
-        grouped_trajs[cast(str, traj.metadata["pipeline"])].append(traj)
-
-    # Find all keys that were different
-    keys_to_print = ["pipeline"]
-    for _, trajs in grouped_trajs.items():
-        keys = dict_diff([traj.metadata for traj in trajs])
-        if len(keys) > 0:
-            keys.remove("name")
-            keys_to_print += keys
-
-    results: list[dict[str, Param]] = []
-    for _pipeline, trajs in grouped_trajs.items():
-        # Iterate over each
-        for traj in trajs:
-            traj_aligned, gt_aligned = stats.align(traj, gt_og)
-            if length is not None and len(traj_aligned) > length:
-                traj_aligned.stamps = traj_aligned.stamps[:length]
-                traj_aligned.poses = traj_aligned.poses[:length]
-                gt_aligned.stamps = gt_aligned.stamps[:length]
-                gt_aligned.poses = gt_aligned.poses[:length]
-            ate = stats.ate(traj_aligned, gt_aligned).summarize(metric)
-            rte = stats.rte(traj_aligned, gt_aligned, window_size).summarize(metric)
-            r = {
-                "name": traj.metadata["name"],
+        # generate colors for visualization
+        colors = distinctipy.get_colors(len(all_trajs) + 1)
+
+    # Iterate over each
+    results: list[dict[str, Any]] = []
+    for index, traj in enumerate(all_trajs):
+        r = cast(ty.Experiment, traj.metadata).to_dict()
+        # flatten pipeline params
+        r.update(r["pipeline_params"])
+        del r["pipeline_params"]
+
+        # add metrics
+        traj_aligned, gt_aligned = stats.align(traj, gt_og)
+        if length is not None and len(traj_aligned) > length:
+            traj_aligned.stamps = traj_aligned.stamps[:length]
+            traj_aligned.poses = traj_aligned.poses[:length]
+            gt_aligned.stamps = gt_aligned.stamps[:length]
+            gt_aligned.poses = gt_aligned.poses[:length]
+        ate = stats.ate(traj_aligned, gt_aligned).summarize(metric)
+        rte = stats.rte(traj_aligned, gt_aligned, window_kind, window_size).summarize(
+            metric
+        )
+
+        r.update(
+            {
                 "RTEt": rte.trans,
                 "RTEr": rte.rot,
                 "ATEt": ate.trans,
                 "ATEr": ate.rot,
-                "length": len(traj_aligned),
             }
-            r.update({k: traj.metadata.get(k, "--") for k in keys_to_print})
-            results.append(r)
-
-            if rr is not None and convert is not None and visualize:
-                rr.log(
-                    cast(str, traj.metadata["name"]),
-                    convert(traj_aligned),
-                    static=True,
-                )
-
-    if sort is not None:
-        results = sorted(results, key=lambda x: x[sort])
-
-    table = Table(
-        title=str(dir),
-        highlight=True,
-        box=box.ROUNDED,
-        min_width=len(str(dir)) + 5,
-    )
+        )
 
-    for key, val in results[0].items():
-        table.add_column(key, justify="right" if isinstance(val, float) else "center")
+        results.append(r)
 
-    for result in results:
-        row = [
-            f"{item:.3f}" if isinstance(item, float) else str(item)
-            for item in result.values()
-        ]
-        table.add_row(*row)
+        if rr is not None and convert is not None and colors is not None and visualize:
+            rr.log(
+                cast(ty.Experiment, traj.metadata).name,
+                convert(traj_aligned, color=colors[index]),
+                static=True,
+            )
 
-    print()
-    Console().print(table)
+    return results
 
 
 def _contains_dir(directory: Path) -> bool:
     return any(directory.is_dir() for directory in directory.glob("*"))
 
 
+def evaluate(
+    directories: list[Path],
+    window_size: Optional[float],
+    window_kind: stats.WindowKind,
+    metric: stats.MetricKind,
+    length: Optional[int],
+    visualize: bool,
+) -> list[dict[str, Any]]:
+    # Collect all bottom level directories
+    bottom_level_dirs: list[Path] = []
+    for directory in directories:
+        for subdir in directory.glob("**/"):
+            if not _contains_dir(subdir):
+                bottom_level_dirs.append(subdir)
+
+    # Compute them all in parallel
+    results = Parallel(n_jobs=-2)(
+        delayed(eval_dataset)(
+            d,
+            visualize,
+            window_kind,
+            window_size,
+            metric,
+            length,
+        )
+        for d in bottom_level_dirs
+    )
+    results = [r for r in results if r is not None]
+
+    return list(itertools.chain.from_iterable(results))
+
+
 @app.command("stats", no_args_is_help=True)
-def eval(
+def evaluate_typer(
     directories: Annotated[
-        list[str], typer.Argument(help="Directory of results to evaluate.")
+        list[Path], typer.Argument(help="Directory of results to evaluate.")
     ],
     visualize: Annotated[
         bool, typer.Option("--visualize", "-v", help="Visualize results.")
     ] = False,
+    # output options
     sort: Annotated[
         str,
-        typer.Option("-s", "--sort", help="Sort results by the name of a column."),
+        typer.Option(
+            "-s",
+            "--sort",
+            help="Sort results by the name of a column.",
+            rich_help_panel="Output options",
+        ),
     ] = "RTEt",
-    window: Annotated[
-        int,
+    reverse: Annotated[
+        bool,
         typer.Option(
-            "-w", "--window", help="Window size for RTE. Defaults to 100 time-steps."
+            "--reverse",
+            "-r",
+            help="Reverse the sorting order. Defaults to False.",
+            rich_help_panel="Output options",
         ),
-    ] = 200,
+    ] = False,
+    # filtering options
+    filter_str: Annotated[
+        Optional[str],
+        typer.Option(
+            "-f",
+            "--filter",
+            help="Python expressions to filter results rows. 'True' rows will be kept. Example: --filter 'RTEt < 0.5'",
+            rich_help_panel="Filtering options",
+        ),
+    ] = None,
+    only_complete: Annotated[
+        bool,
+        typer.Option(
+            "--only-complete",
+            help="Only show results for trajectories that completed.",
+            rich_help_panel="Filtering options",
+        ),
+    ] = False,
+    only_failed: Annotated[
+        bool,
+        typer.Option(
+            "--only-failed",
+            help="Only show results for trajectories that failed.",
+            rich_help_panel="Filtering options",
+        ),
+    ] = False,
+    hide_columns: Annotated[
+        list[str],
+        typer.Option(
+            "-s",
+            "--show-columns",
+            help="Comma-separated list of columns to show.",
+            rich_help_panel="Output options",
+        ),
+    ] = ["pipeline_version", "total_elapsed", "pipeline"],
+    print_columns: Annotated[
+        bool,
+        typer.Option(
+            "--print-columns",
+            help="Print the names of all available columns.",
+            rich_help_panel="Output options",
+        ),
+    ] = False,
+    # metric options
+    window_size: Annotated[
+        Optional[float],
+        typer.Option(
+            "-w",
+            "--window-size",
+            help="Window size for RTE. Defaults to 100 time steps for time windows, 10 meters for distance windows.",
+            rich_help_panel="Metric options",
+        ),
+    ] = None,
+    window_kind: Annotated[
+        stats.WindowKind,
+        typer.Option(
+            "-k",
+            "--window-kind",
+            help="Kind of window to use for RTE. Defaults to time.",
+            rich_help_panel="Metric options",
+        ),
+    ] = stats.WindowKind.time,
     metric: Annotated[
         stats.MetricKind,
         typer.Option(
             "--metric",
             "-m",
             help="Metric to use for ATE/RTE computation. Defaults to sse.",
+            rich_help_panel="Metric options",
         ),
     ] = stats.MetricKind.sse,
     length: Annotated[
         Optional[int],
         typer.Option(
-            "-l", "--length", help="Specify subset of trajectory to evaluate."
+            "-l",
+            "--length",
+            help="Specify subset of trajectory to evaluate.",
+            rich_help_panel="Metric options",
         ),
     ] = None,
-):
+) -> None:
     """
     Evaluate the results of experiments.
     """
+    # ------------------------- Process all inputs ------------------------- #
+    # Parse some of the options
+    if only_complete and only_failed:
+        raise typer.BadParameter(
+            "Can only use one of --only-complete, --only-incomplete, or --only-failed."
+        )
 
-    directories_path = [Path(d) for d in directories]
+    # Parse the filtering options
+    filter_method: Callable[[dict[str, Any]], bool]
+    if filter_str is None:
+        filter_method = lambda r: True  # noqa: E731
+    else:
+        filter_method = lambda r: eval(  # noqa: E731
+            filter_str,
+            {"__builtins__": None},
+            {"np": np, **r},
+        )
+
+    original_filter = filter_method
+    if only_complete:
+        filter_method = lambda r: original_filter(r) and r["status"] == "complete"  # noqa: E731
+    elif only_failed:
+        filter_method = lambda r: original_filter(r) and r["status"] == "fail"  # noqa: E731
+
+    match window_kind:
+        case stats.WindowKind.distance:
+            if window_size is None:
+                window_size = 10
+            words = "meters"
+        case stats.WindowKind.time:
+            if window_size is None:
+                window_size = 200
+            words = "time steps"
 
     c = Console()
-    c.print(f"Evaluating RTE over a window of size {window}, using metric {metric}.")
+    c.print(
+        f"Evaluating RTE over a window of {window_size} {words}, using metric {metric}."
+    )
 
-    # Collect all bottom level directories
-    bottom_level_dirs: list[Path] = []
-    for directory in directories_path:
-        for subdir in directory.glob("**/"):
-            if not _contains_dir(subdir):
-                bottom_level_dirs.append(subdir)
+    # ------------------------- Compute all results ------------------------- #
+    results = evaluate(
+        directories,
+        window_size,
+        window_kind,
+        metric,
+        length,
+        visualize,
+    )
+
+    # ------------------------- Filter all results ------------------------- #
+    try:
+        results = [r for r in results if filter_method(r)]
+    except Exception as e:
+        print_warning(f"Error filtering results: {e}")
+
+    # convert to polars dataframe for easier processing
+    if len(results) == 0:
+        print_warning("No results found.")
+        return
+
+    df = pl.DataFrame(results)
+
+    # clean up timing
+    df = df.with_columns(
+        ((pl.col("sequence_length") / pl.col("total_elapsed")).alias("Hz"))
+    )
+    df = df.rename({"max_step_elapsed": "Max (s)"})
+
+    # print columns if requested
+    if print_columns:
+        c.print("Available columns:")
+        for col in df.columns:
+            c.print(f" - {col}")
+        return
+
+    # delete unneeded columns
+    remove_columns = [col for col in df.columns if df[col].drop_nulls().n_unique() == 1]
+    remove_columns.extend([col for col in hide_columns if col in df.columns])
+    df = df.drop(remove_columns)
+
+    # sort if requested
+    if sort not in df.columns:
+        print_warning(f"Column {sort} not found, cannot sort.")
+    else:
+        df = df.sort(sort, descending=reverse)
+
+    # ------------------------- Print ------------------------- #
+    # Print sequence by sequence
+    for sequence in df["sequence"].unique():
+        df_sequence = df.filter(pl.col("sequence") == sequence)
+        df_sequence = df_sequence.drop("sequence")
+        if df_sequence.is_empty():
+            continue
+
+        table = Table(
+            title=f"Results for {sequence}",
+            box=box.ROUNDED,
+            highlight=True,
+            # show_lines=True,
+            # header_style="bold magenta",
+            # min_width =
+        )
+
+        for col in df_sequence.columns:
+            table.add_column(
+                col,
+                justify="right" if df_sequence[col].dtype is pl.Float64 else "left",
+                no_wrap=True,
+            )
+
+        for row in df_sequence.iter_rows():
+            table.add_row(
+                *[f"{x:.3f}" if isinstance(x, float) else str(x) for x in row]
+            )
 
-    for d in bottom_level_dirs:
-        eval_dataset(d, visualize, sort, window, metric, length)
+        c.print(table)
+        c.print("\n")
diff --git a/python/evalio/types/base.py b/python/evalio/types/base.py
index af6352e2..9aab5f45 100644
--- a/python/evalio/types/base.py
+++ b/python/evalio/types/base.py
@@ -205,7 +205,7 @@ def from_tum(path: Path) -> "Trajectory":
         return Trajectory.from_csv(path, ["sec", "x", "y", "z", "qx", "qy", "qz", "qw"])
 
     @staticmethod
-    def from_file(path: Path) -> Trajectory | FailedMetadataParse:
+    def from_file(path: Path) -> Trajectory | FailedMetadataParse | FileNotFoundError:
         """Load a saved evalio trajectory from file.
 
         Works identically to [from_tum][evalio.types.Trajectory.from_tum], but also loads metadata from the file.
@@ -216,6 +216,9 @@ def from_file(path: Path) -> Trajectory | FailedMetadataParse:
         Returns:
             Trajectory: Loaded trajectory with metadata, stamps, and poses.
         """
+        if not path.exists():
+            return FileNotFoundError(f"File {path} does not exist.")
+
         with open(path) as file:
             metadata_filter = filter(
                 lambda row: row[0] == "#" and not row.startswith("# timestamp,"), file
@@ -293,6 +296,8 @@ def open(self, path: Optional[Path] = None):
                 "Trajectory.open: No metadata or path provided, cannot set metadata file."
             )
             return
+
+        path.parent.mkdir(parents=True, exist_ok=True)
         self._file = path.open("w")
         self._csv_writer = csv.writer(self._file)
         self._write()
diff --git a/uv.lock b/uv.lock
index 644f69a0..da07a255 100644
--- a/uv.lock
+++ b/uv.lock
@@ -25,15 +25,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916, upload-time = "2025-03-17T00:02:52.713Z" },
 ]
 
-[[package]]
-name = "argcomplete"
-version = "3.6.2"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/16/0f/861e168fc813c56a78b35f3c30d91c6757d1fd185af1110f1aec784b35d0/argcomplete-3.6.2.tar.gz", hash = "sha256:d0519b1bc867f5f4f4713c41ad0aba73a4a5f007449716b16f385f2166dc6adf", size = 73403, upload-time = "2025-04-03T04:57:03.52Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/31/da/e42d7a9d8dd33fa775f467e4028a47936da2f01e4b0e561f9ba0d74cb0ca/argcomplete-3.6.2-py3-none-any.whl", hash = "sha256:65b3133a29ad53fb42c48cf5114752c7ab66c1c38544fdf6460f450c09b42591", size = 43708, upload-time = "2025-04-03T04:57:01.591Z" },
-]
-
 [[package]]
 name = "attrs"
 version = "25.3.0"
@@ -294,10 +285,11 @@ name = "evalio"
 version = "0.3.0"
 source = { editable = "." }
 dependencies = [
-    { name = "argcomplete" },
     { name = "distinctipy" },
     { name = "gdown" },
+    { name = "joblib" },
     { name = "numpy" },
+    { name = "polars" },
     { name = "pyyaml" },
     { name = "rapidfuzz" },
     { name = "rosbags" },
@@ -316,6 +308,7 @@ dev = [
     { name = "bump-my-version" },
     { name = "cmake" },
     { name = "compdb" },
+    { name = "joblib-stubs" },
     { name = "mike" },
     { name = "mkdocs" },
     { name = "mkdocs-gen-files" },
@@ -333,10 +326,11 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "argcomplete", specifier = ">=3.3.0" },
     { name = "distinctipy", specifier = ">=1.3.4" },
     { name = "gdown", specifier = ">=5.2.0" },
+    { name = "joblib", specifier = ">=1.5.2" },
     { name = "numpy" },
+    { name = "polars", specifier = ">=1.33.1" },
     { name = "pyyaml", specifier = ">=6.0" },
     { name = "rapidfuzz", specifier = ">=3.12.2" },
     { name = "rerun-sdk", marker = "extra == 'vis'", specifier = ">=0.23" },
@@ -352,6 +346,7 @@ dev = [
     { name = "bump-my-version", specifier = ">=1.1.1" },
     { name = "cmake", specifier = "<4.0.0" },
     { name = "compdb", specifier = ">=0.2.0" },
+    { name = "joblib-stubs", specifier = ">=1.5.2.0.20250831" },
     { name = "mike", specifier = ">=2.1.3" },
     { name = "mkdocs", specifier = ">=1.6.1" },
     { name = "mkdocs-gen-files", specifier = ">=0.5.0" },
@@ -503,6 +498,27 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" },
 ]
 
+[[package]]
+name = "joblib"
+version = "1.5.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e8/5d/447af5ea094b9e4c4054f82e223ada074c552335b9b4b2d14bd9b35a67c4/joblib-1.5.2.tar.gz", hash = "sha256:3faa5c39054b2f03ca547da9b2f52fde67c06240c31853f306aea97f13647b55", size = 331077, upload-time = "2025-08-27T12:15:46.575Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/1e/e8/685f47e0d754320684db4425a0967f7d3fa70126bffd76110b7009a0090f/joblib-1.5.2-py3-none-any.whl", hash = "sha256:4e1f0bdbb987e6d843c70cf43714cb276623def372df3c22fe5266b2670bc241", size = 308396, upload-time = "2025-08-27T12:15:45.188Z" },
+]
+
+[[package]]
+name = "joblib-stubs"
+version = "1.5.2.0.20250831"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/88/80/2e0ea0e43642ab69b33ad0d39c73378f52cc8b961b3dfc2519d53639e518/joblib_stubs-1.5.2.0.20250831.tar.gz", hash = "sha256:1b419c5b5238cbfd2759ef2e463ae3399ba1d3a6cbdc213a9ac339e6312b21ef", size = 19886, upload-time = "2025-08-31T11:39:17.607Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/57/f7/f73413f8c13d208f291d1430528b415331320868c352b6e8e5442c5ad053/joblib_stubs-1.5.2.0.20250831-py3-none-any.whl", hash = "sha256:2b75f04fbb98975d207cc3d7b686c538ee1b2a749cb63c035ce41ff97a0eb4bc", size = 36252, upload-time = "2025-08-31T11:39:16.447Z" },
+]
+
 [[package]]
 name = "lz4"
 version = "4.4.4"
@@ -959,6 +975,20 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669", size = 20556, upload-time = "2024-04-20T21:34:40.434Z" },
 ]
 
+[[package]]
+name = "polars"
+version = "1.33.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/85/da/8246f1d69d7e49f96f0c5529057a19af1536621748ef214bbd4112c83b8e/polars-1.33.1.tar.gz", hash = "sha256:fa3fdc34eab52a71498264d6ff9b0aa6955eb4b0ae8add5d3cb43e4b84644007", size = 4822485, upload-time = "2025-09-09T08:37:49.062Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/19/79/c51e7e1d707d8359bcb76e543a8315b7ae14069ecf5e75262a0ecb32e044/polars-1.33.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:3881c444b0f14778ba94232f077a709d435977879c1b7d7bd566b55bd1830bb5", size = 39132875, upload-time = "2025-09-09T08:36:38.609Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/15/1094099a1b9cb4fbff58cd8ed3af8964f4d22a5b682ea0b7bb72bf4bc3d9/polars-1.33.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:29200b89c9a461e6f06fc1660bc9c848407640ee30fe0e5ef4947cfd49d55337", size = 35638783, upload-time = "2025-09-09T08:36:43.748Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/b9/9ac769e4d8e8f22b0f2e974914a63dd14dec1340cd23093de40f0d67d73b/polars-1.33.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:444940646e76342abaa47f126c70e3e40b56e8e02a9e89e5c5d1c24b086db58a", size = 39742297, upload-time = "2025-09-09T08:36:47.132Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/26/4c5da9f42fa067b2302fe62bcbf91faac5506c6513d910fae9548fc78d65/polars-1.33.1-cp39-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:094a37d06789286649f654f229ec4efb9376630645ba8963b70cb9c0b008b3e1", size = 36684940, upload-time = "2025-09-09T08:36:50.561Z" },
+    { url = "https://files.pythonhosted.org/packages/06/a6/dc535da476c93b2efac619e04ab81081e004e4b4553352cd10e0d33a015d/polars-1.33.1-cp39-abi3-win_amd64.whl", hash = "sha256:c9781c704432a2276a185ee25898aa427f39a904fbe8fde4ae779596cdbd7a9e", size = 39456676, upload-time = "2025-09-09T08:36:54.612Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/4e/a4300d52dd81b58130ccadf3873f11b3c6de54836ad4a8f32bac2bd2ba17/polars-1.33.1-cp39-abi3-win_arm64.whl", hash = "sha256:c3cfddb3b78eae01a218222bdba8048529fef7e14889a71e33a5198644427642", size = 35445171, upload-time = "2025-09-09T08:36:58.043Z" },
+]
+
 [[package]]
 name = "prompt-toolkit"
 version = "3.0.51"

From 82268006fa2411b0b513e617ecb3f29ca0c9779f Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Tue, 30 Sep 2025 21:44:39 -0400
Subject: [PATCH 11/40] Fix some of pipeline parsing

---
 python/evalio/pipelines/parser.py |  4 ++--
 tests/test_parsing.py             | 19 ++++++++++---------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/python/evalio/pipelines/parser.py b/python/evalio/pipelines/parser.py
index e92fa6a8..76d86bd1 100644
--- a/python/evalio/pipelines/parser.py
+++ b/python/evalio/pipelines/parser.py
@@ -189,7 +189,7 @@ def parse_config(
         # figure out name of pipeline
         if "pipeline" not in p:
             return InvalidPipelineConfig(f"Need pipeline: {str(p)}")
-        pipe_name = cast(str, p["pipeline"])
+        pipe_name = cast(str, p.pop("pipeline"))
 
         # figure out the name
         name = p.pop("name", pipe_name)
@@ -205,7 +205,7 @@ def parse_config(
 
         # Handle sweeps
         if "sweep" in p:
-            sweep = cast(dict[str, Param], p.pop("sweep"))
+            sweep = cast(dict[str, Param], params.pop("sweep"))
             return _sweep(sweep, name, pipe, params)
         else:
             err = validate_params(pipe, params)
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index cef2cfa0..e1bcd9f3 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -54,22 +54,23 @@ def default_params() -> dict[str, Param]:
 pl.register_pipeline(FakePipeline)
 
 # fmt: off
+dp = FakePipeline.default_params()
 PIPELINES: list[Any] = [
     # good ones
-    ("fake", [("fake", FakePipeline, {})]),
-    ({"pipeline": "fake"}, [("fake", FakePipeline, {})]),
-    ({"name": "test", "pipeline": "fake"}, [("test", FakePipeline, {})]),
-    ({"pipeline": "fake", "param1": 5}, [("fake", FakePipeline, {"param1": 5})]),
-    (["fake", {"pipeline": "fake", "param1": 3}], [("fake", FakePipeline, {}), ("fake", FakePipeline, {"param1": 3})]),
+    ("fake", [("fake", FakePipeline, dp)]),
+    ({"pipeline": "fake"}, [("fake", FakePipeline, dp)]),
+    ({"name": "test", "pipeline": "fake"}, [("test", FakePipeline, dp)]),
+    ({"pipeline": "fake", "param1": 5}, [("fake", FakePipeline, dp | {"param1": 5})]),
+    (["fake", {"pipeline": "fake", "param1": 3}], [("fake", FakePipeline, dp), ("fake", FakePipeline, dp | {"param1": 3})]),
     ({"pipeline": "fake", "sweep": {"param1": [1, 2, 3]}}, [
-        ("fake__param1-1", FakePipeline, {"param1": 1}),
-        ("fake__param1-2", FakePipeline, {"param1": 2}),
-        ("fake__param1-3", FakePipeline, {"param1": 3}),
+        ("fake__param1-1", FakePipeline, dp | {"param1": 1}),
+        ("fake__param1-2", FakePipeline, dp | {"param1": 2}),
+        ("fake__param1-3", FakePipeline, dp | {"param1": 3}),
     ]),
     # bad ones
     ("unknown", pl.PipelineNotFound("unknown")),
     ({"pipeline": "unknown"}, pl.PipelineNotFound("unknown")),
-    ({"param1": 5}, pl.InvalidPipelineConfig("Need pipeline name: {'param1': 5}")), # type: ignore
+    ({"param1": 5}, pl.InvalidPipelineConfig("Need pipeline: {'param1': 5}")), # type: ignore
     ({"pipeline": "fake", "param3": 10}, pl.UnusedPipelineParam("param3", "fake")),
     ({"pipeline": "fake", "param1": "wrong_type"}, pl.InvalidPipelineParamType("param1", int, str)),
     ({"pipeline": "fake", "sweep": {"param1": [1.0, 2, 3]}}, pl.InvalidPipelineParamType("param1", int, float)),

From a4b45f2d23aa8925734e39240d55259610f278b8 Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Tue, 30 Sep 2025 22:00:33 -0400
Subject: [PATCH 12/40] Add generics to Trajectory to help with metadata types

---
 python/evalio/cli/run.py       |  7 +++----
 python/evalio/cli/stats.py     | 12 ++++++------
 python/evalio/datasets/base.py |  5 +++--
 python/evalio/rerun.py         | 18 +++++++++++++++---
 python/evalio/stats.py         | 27 ++++++++++++++++-----------
 python/evalio/types/base.py    | 15 +++++++++++----
 tests/test_csv_loading.py      |  4 ++--
 tests/test_io.py               | 10 +++-------
 8 files changed, 59 insertions(+), 39 deletions(-)

diff --git a/python/evalio/cli/run.py b/python/evalio/cli/run.py
index 4cecd79e..6cff857e 100644
--- a/python/evalio/cli/run.py
+++ b/python/evalio/cli/run.py
@@ -361,9 +361,8 @@ def run_single(
                 break
 
     loop.close()
-    if isinstance(traj.metadata, ty.Experiment):
-        traj.metadata.status = ty.ExperimentStatus.Complete
-        traj.metadata.total_elapsed = time_total
-        traj.metadata.max_step_elapsed = time_max
+    traj.metadata.status = ty.ExperimentStatus.Complete
+    traj.metadata.total_elapsed = time_total
+    traj.metadata.max_step_elapsed = time_max
     traj.rewrite()
     traj.close()
diff --git a/python/evalio/cli/stats.py b/python/evalio/cli/stats.py
index 32035e7e..0a737c40 100644
--- a/python/evalio/cli/stats.py
+++ b/python/evalio/cli/stats.py
@@ -31,8 +31,8 @@ def eval_dataset(
     length: Optional[int],
 ) -> Optional[list[dict[str, Any]]]:
     # Load all trajectories
-    gt_og: Optional[ty.Trajectory] = None
-    all_trajs: list[ty.Trajectory] = []
+    gt_og: Optional[ty.Trajectory[ty.GroundTruth]] = None
+    all_trajs: list[ty.Trajectory[ty.Experiment]] = []
     for file_path in dir.glob("*.csv"):
         traj = ty.Trajectory.from_file(file_path)
         if not isinstance(traj, ty.Trajectory):
@@ -42,9 +42,9 @@ def eval_dataset(
             if gt_og is not None:
                 print_warning(f"Multiple ground truths found in {dir}, skipping.")
                 continue
-            gt_og = traj
+            gt_og = cast(ty.Trajectory[ty.GroundTruth], traj)
         elif isinstance(traj.metadata, ty.Experiment):
-            all_trajs.append(traj)
+            all_trajs.append(cast(ty.Trajectory[ty.Experiment], traj))
 
     if gt_og is None:
         print_warning(f"No ground truth found in {dir}, skipping.")
@@ -82,7 +82,7 @@ def eval_dataset(
     # Iterate over each
     results: list[dict[str, Any]] = []
     for index, traj in enumerate(all_trajs):
-        r = cast(ty.Experiment, traj.metadata).to_dict()
+        r = traj.metadata.to_dict()
         # flatten pipeline params
         r.update(r["pipeline_params"])
         del r["pipeline_params"]
@@ -112,7 +112,7 @@ def eval_dataset(
 
         if rr is not None and convert is not None and colors is not None and visualize:
             rr.log(
-                cast(ty.Experiment, traj.metadata).name,
+                traj.metadata.name,
                 convert(traj_aligned, color=colors[index]),
                 static=True,
             )
diff --git a/python/evalio/datasets/base.py b/python/evalio/datasets/base.py
index 9190feb6..dcf5d36e 100644
--- a/python/evalio/datasets/base.py
+++ b/python/evalio/datasets/base.py
@@ -2,7 +2,7 @@
 from enum import StrEnum
 from itertools import islice
 from pathlib import Path
-from typing import Iterable, Iterator, Optional, Sequence, Union
+from typing import Iterable, Iterator, Optional, Sequence, Union, cast
 
 from evalio._cpp.types import (  # type: ignore
     SE3,
@@ -210,7 +210,7 @@ def is_downloaded(self) -> bool:
 
         return True
 
-    def ground_truth(self) -> Trajectory:
+    def ground_truth(self) -> Trajectory[GroundTruth]:
         """Get the ground truth trajectory in the **IMU** frame, rather than the ground truth frame as returned in [ground_truth_raw][evalio.datasets.Dataset.ground_truth_raw].
 
         Returns:
@@ -224,6 +224,7 @@ def ground_truth(self) -> Trajectory:
             gt_o_T_gt_i = gt_traj.poses[i]
             gt_traj.poses[i] = gt_o_T_gt_i * gt_T_imu
 
+        gt_traj = cast(Trajectory[GroundTruth], gt_traj)
         gt_traj.metadata = GroundTruth(sequence=self.full_name)
 
         return gt_traj
diff --git a/python/evalio/rerun.py b/python/evalio/rerun.py
index 59f29796..b4b6d2f5 100644
--- a/python/evalio/rerun.py
+++ b/python/evalio/rerun.py
@@ -1,5 +1,6 @@
 from dataclasses import dataclass
 from typing import Any, Literal, Optional, Sequence, TypedDict, cast, overload
+from typing_extensions import TypeVar
 from uuid import UUID, uuid4
 
 import distinctipy
@@ -10,7 +11,16 @@
 from evalio.datasets import Dataset
 from evalio.pipelines import Pipeline
 from evalio.stats import _check_overstep
-from evalio.types import SE3, LidarMeasurement, LidarParams, Point, Stamp, Trajectory
+from evalio.types import (
+    SE3,
+    GroundTruth,
+    LidarMeasurement,
+    LidarParams,
+    Metadata,
+    Point,
+    Stamp,
+    Trajectory,
+)
 from evalio.utils import print_warning
 
 
@@ -78,7 +88,7 @@ def __init__(self, args: VisArgs, pipeline_names: list[str]):
 
             # To be set during new_recording
             self.lidar_params: Optional[LidarParams] = None
-            self.gt: Optional[Trajectory] = None
+            self.gt: Optional[Trajectory[GroundTruth]] = None
             self.pipeline_names = pipeline_names
 
             # To be found during log
@@ -326,9 +336,11 @@ def convert(
         """
         ...
 
+    M = TypeVar("M", bound=Metadata | None)
+
     @overload
     def convert(
-        obj: Trajectory,
+        obj: Trajectory[M],
         color: Optional[tuple[int, int, int] | tuple[float, float, float]] = None,
     ) -> rr.Points3D:
         """Convert a Trajectory a rerun Points3D.
diff --git a/python/evalio/stats.py b/python/evalio/stats.py
index 1b770cf6..b3d3a256 100644
--- a/python/evalio/stats.py
+++ b/python/evalio/stats.py
@@ -1,7 +1,8 @@
 from enum import StrEnum, auto
+from typing_extensions import TypeVar
 
 from evalio.utils import print_warning
-from .types import Stamp, Trajectory, SE3
+from .types import Stamp, Trajectory, SE3, Metadata
 
 from dataclasses import dataclass
 
@@ -100,9 +101,13 @@ def median(self) -> Metric:
         )
 
 
+M1 = TypeVar("M1", bound=Metadata | None)
+M2 = TypeVar("M2", bound=Metadata | None)
+
+
 def align(
-    traj: Trajectory, gt: Trajectory, in_place: bool = False
-) -> tuple[Trajectory, Trajectory]:
+    traj: Trajectory[M1], gt: Trajectory[M2], in_place: bool = False
+) -> tuple[Trajectory[M1], Trajectory[M2]]:
     """Align the trajectories both spatially and temporally.
 
     The resulting trajectories will be have the same origin as the second ("gt") trajectory.
@@ -123,7 +128,7 @@ def align(
     return traj, gt
 
 
-def align_poses(traj: Trajectory, other: Trajectory):
+def align_poses(traj: Trajectory[M1], other: Trajectory[M2]):
     """Align the trajectory in place to another trajectory. Operates in place.
 
     This results in the current trajectory having an identical first pose to the other trajectory.
@@ -141,7 +146,7 @@ def align_poses(traj: Trajectory, other: Trajectory):
         traj.poses[i] = delta * traj.poses[i]
 
 
-def align_stamps(traj1: Trajectory, traj2: Trajectory):
+def align_stamps(traj1: Trajectory[M1], traj2: Trajectory[M2]):
     """Select the closest poses in traj1 and traj2. Operates in place.
 
     Does this by finding the higher frame rate trajectory and subsampling it to the closest poses of the other one.
@@ -175,7 +180,7 @@ def align_stamps(traj1: Trajectory, traj2: Trajectory):
     traj_1_dt = (traj1.stamps[-1] - traj1.stamps[0]).to_sec() / len(traj1.stamps)
     traj_2_dt = (traj2.stamps[-1] - traj2.stamps[0]).to_sec() / len(traj2.stamps)
     if traj_1_dt > traj_2_dt:
-        traj1, traj2 = traj2, traj1
+        traj1, traj2 = traj2, traj1  # type: ignore
         swapped = True
 
     # Align the two trajectories by subsampling keeping traj1 stamps
@@ -202,7 +207,7 @@ def align_stamps(traj1: Trajectory, traj2: Trajectory):
     traj1.poses = traj1_poses
 
     if swapped:
-        traj1, traj2 = traj2, traj1
+        traj1, traj2 = traj2, traj1  # type: ignore
 
 
 def _compute_metric(gts: list[SE3], poses: list[SE3]) -> Error:
@@ -228,7 +233,7 @@ def _compute_metric(gts: list[SE3], poses: list[SE3]) -> Error:
     return Error(rot=error_r, trans=error_t)
 
 
-def _check_aligned(traj: Trajectory, gt: Trajectory) -> bool:
+def _check_aligned(traj: Trajectory[M1], gt: Trajectory[M2]) -> bool:
     """Check if the two trajectories are aligned.
 
     This is done by checking if the first poses are identical, and if there's the same number of poses in both trajectories.
@@ -250,7 +255,7 @@ def _check_aligned(traj: Trajectory, gt: Trajectory) -> bool:
     )
 
 
-def ate(traj: Trajectory, gt: Trajectory) -> Error:
+def ate(traj: Trajectory[M1], gt: Trajectory[M2]) -> Error:
     """Compute the Absolute Trajectory Error (ATE) between two trajectories.
 
     Will check if the two trajectories are aligned and if not, will align them.
@@ -271,8 +276,8 @@ def ate(traj: Trajectory, gt: Trajectory) -> Error:
 
 
 def rte(
-    traj: Trajectory,
-    gt: Trajectory,
+    traj: Trajectory[M1],
+    gt: Trajectory[M2],
     kind: WindowKind = WindowKind.time,
     window: Optional[float | int] = None,
 ) -> Error:
diff --git a/python/evalio/types/base.py b/python/evalio/types/base.py
index 9aab5f45..2d49e0e4 100644
--- a/python/evalio/types/base.py
+++ b/python/evalio/types/base.py
@@ -11,12 +11,13 @@
 from _csv import Writer
 from enum import Enum
 from io import TextIOWrapper
+from typing_extensions import TypeVar
 from evalio.utils import print_warning
 import numpy as np
 import yaml
 
 from pathlib import Path
-from typing import Any, ClassVar, Optional, Self
+from typing import Any, ClassVar, Generic, Optional, Self, cast
 
 from evalio._cpp.types import (  # type: ignore
     SE3,
@@ -93,13 +94,16 @@ class GroundTruth(Metadata):
     """Dataset used to run the experiment."""
 
 
+M = TypeVar("M", bound=Metadata | None, default=None)
+
+
 @dataclass(kw_only=True)
-class Trajectory:
+class Trajectory(Generic[M]):
     stamps: list[Stamp] = field(default_factory=list)
     """List of timestamps for each pose."""
     poses: list[SE3] = field(default_factory=list)
     """List of poses, in the same order as the timestamps."""
-    metadata: Optional[Metadata] = None
+    metadata: M = None  # type: ignore
     """Metadata associated with the trajectory, such as the dataset name or other information."""
     _file: Optional[TextIOWrapper] = None
     _csv_writer: Optional[Writer] = None
@@ -205,7 +209,9 @@ def from_tum(path: Path) -> "Trajectory":
         return Trajectory.from_csv(path, ["sec", "x", "y", "z", "qx", "qy", "qz", "qw"])
 
     @staticmethod
-    def from_file(path: Path) -> Trajectory | FailedMetadataParse | FileNotFoundError:
+    def from_file(
+        path: Path,
+    ) -> Trajectory[Metadata] | FailedMetadataParse | FileNotFoundError:
         """Load a saved evalio trajectory from file.
 
         Works identically to [from_tum][evalio.types.Trajectory.from_tum], but also loads metadata from the file.
@@ -236,6 +242,7 @@ def from_file(path: Path) -> Trajectory | FailedMetadataParse | FileNotFoundErro
             path,
             fieldnames=["sec", "x", "y", "z", "qx", "qy", "qz", "qw"],
         )
+        trajectory = cast(Trajectory[Metadata], trajectory)
         trajectory.metadata = metadata
 
         return trajectory
diff --git a/tests/test_csv_loading.py b/tests/test_csv_loading.py
index ee0044af..ee17722d 100644
--- a/tests/test_csv_loading.py
+++ b/tests/test_csv_loading.py
@@ -72,7 +72,7 @@ def serialize_stamp(self, stamp: Stamp) -> str:
                 return f"{stamp.sec}, {stamp.nsec}"
 
 
-def fake_groundtruth() -> Trajectory:
+def fake_groundtruth() -> Trajectory[GroundTruth]:
     stamps = [
         Stamp.from_nsec(i + np.random.randint(-500, 500))
         for i in range(1_000, 10_000, 1_000)
@@ -81,7 +81,7 @@ def fake_groundtruth() -> Trajectory:
     return Trajectory(stamps=stamps, poses=poses, metadata=GroundTruth(sequence="fake"))
 
 
-def serialize_gt(gt: Trajectory, style: StampStyle) -> list[str]:
+def serialize_gt(gt: Trajectory[GroundTruth], style: StampStyle) -> list[str]:
     def serialize_se3(se3: SE3) -> str:
         return f"{se3.trans[0]}, {se3.trans[1]}, {se3.trans[2]}, {se3.rot.qx}, {se3.rot.qy}, {se3.rot.qz}, {se3.rot.qw}"
 
diff --git a/tests/test_io.py b/tests/test_io.py
index 93937ada..b6b45d07 100644
--- a/tests/test_io.py
+++ b/tests/test_io.py
@@ -36,8 +36,7 @@ def test_trajectory_serde(tmp_path: Path):
         metadata=make_exp(),
     )
 
-    if traj.metadata is not None:
-        traj.metadata.file = path
+    traj.metadata.file = path
 
     traj.to_file(path)
 
@@ -56,9 +55,7 @@ def test_trajectory_incremental_serde(tmp_path: Path):
         poses=[ty.SE3.exp(np.random.rand(6)) for _ in range(5)],
         metadata=make_exp(),
     )
-
-    if traj.metadata is not None:
-        traj.metadata.file = path
+    traj.metadata.file = path
 
     # poses are automatically written as they are added
     traj.open(path)
@@ -70,8 +67,7 @@ def test_trajectory_incremental_serde(tmp_path: Path):
 
     # must trigger entire rewrite to update metadata
     traj.open(path)
-    if traj.metadata is not None:
-        traj.metadata.sequence = "random_name"  # type: ignore
+    traj.metadata.sequence = "random_name"  # type: ignore
     traj.rewrite()
     traj.close()
 

From 8935da9c3374f2880be74cc04106ffa6769b805b Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Tue, 30 Sep 2025 22:05:37 -0400
Subject: [PATCH 13/40] Remove a bunch of type: ignores

---
 python/evalio/cli/ls.py                      | 37 ++++++++++----------
 python/evalio/datasets/multi_campus.py       |  2 +-
 python/evalio/datasets/newer_college_2020.py |  2 +-
 python/evalio/stats.py                       |  2 +-
 tests/test_io.py                             |  2 +-
 tests/test_parsing.py                        |  2 +-
 6 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/python/evalio/cli/ls.py b/python/evalio/cli/ls.py
index 21ab6848..a7fb04a3 100644
--- a/python/evalio/cli/ls.py
+++ b/python/evalio/cli/ls.py
@@ -1,5 +1,5 @@
 from enum import StrEnum, auto
-from typing import Annotated, Optional, TypeVar
+from typing import Annotated, Literal, Optional, TypeVar, TypedDict
 
 import typer
 from rapidfuzz.process import extract_iter
@@ -83,6 +83,9 @@ def ls(
     """
     List dataset and pipeline information
     """
+    ColOpts = TypedDict("ColOpts", {"vertical": Literal["top", "middle", "bottom"]})
+    col_opts: ColOpts = {"vertical": "middle"}
+
     if kind == Kind.datasets:
         # Search for datasets using rapidfuzz
         # TODO: Make it search through sequences as well?
@@ -177,20 +180,19 @@ def ls(
             highlight=True,
             box=box.ROUNDED,
         )
-        col_opts = {"vertical": "middle"}
 
-        table.add_column("Name", justify="center", **col_opts)  # type: ignore
+        table.add_column("Name", justify="center", **col_opts)
         if not quiet:
-            table.add_column("Sequences", justify="right", **col_opts)  # type: ignore
-            table.add_column("DL", justify="right", **col_opts)  # type: ignore
-        table.add_column("Size", justify="center", **col_opts)  # type: ignore
+            table.add_column("Sequences", justify="right", **col_opts)
+            table.add_column("DL", justify="right", **col_opts)
+        table.add_column("Size", justify="center", **col_opts)
         if not quiet:
-            table.add_column("Len", justify="center", **col_opts)  # type: ignore
-        table.add_column("Env", justify="center", **col_opts)  # type: ignore
-        table.add_column("Vehicle", justify="center", **col_opts)  # type: ignore
-        table.add_column("IMU", justify="center", **col_opts)  # type: ignore
-        table.add_column("LiDAR", justify="center", **col_opts)  # type: ignore
-        table.add_column("Info", justify="center", **col_opts)  # type: ignore
+            table.add_column("Len", justify="center", **col_opts)
+        table.add_column("Env", justify="center", **col_opts)
+        table.add_column("Vehicle", justify="center", **col_opts)
+        table.add_column("IMU", justify="center", **col_opts)
+        table.add_column("LiDAR", justify="center", **col_opts)
+        table.add_column("Info", justify="center", **col_opts)
 
         for i in range(len(all_info["Name"])):
             row_info = [all_info[c.header][i] for c in table.columns]  # type: ignore
@@ -253,14 +255,13 @@ def ls(
             highlight=True,
             box=box.ROUNDED,
         )
-        col_opts = {"vertical": "middle"}
 
-        table.add_column("Name", justify="center", **col_opts)  # type: ignore
-        table.add_column("Version", justify="center", **col_opts)  # type: ignore
+        table.add_column("Name", justify="center", **col_opts)
+        table.add_column("Version", justify="center", **col_opts)
         if not quiet:
-            table.add_column("Params", justify="right", **col_opts)  # type: ignore
-            table.add_column("Default", justify="left", **col_opts)  # type: ignore
-        table.add_column("Info", justify="center", **col_opts)  # type: ignore
+            table.add_column("Params", justify="right", **col_opts)
+            table.add_column("Default", justify="left", **col_opts)
+        table.add_column("Info", justify="center", **col_opts)
 
         for i in range(len(all_info["Name"])):
             row_info = [all_info[c.header][i] for c in table.columns]  # type: ignore
diff --git a/python/evalio/datasets/multi_campus.py b/python/evalio/datasets/multi_campus.py
index 0a087456..43534bfd 100644
--- a/python/evalio/datasets/multi_campus.py
+++ b/python/evalio/datasets/multi_campus.py
@@ -295,7 +295,7 @@ def download(self):
             "tuhh_night_09": "1xr5dTBydbjIhE42hNdELklruuhxgYkld",
         }[self.seq_name]
 
-        import gdown  # type: ignore
+        import gdown
 
         print(f"Downloading to {self.folder}...")
         self.folder.mkdir(parents=True, exist_ok=True)
diff --git a/python/evalio/datasets/newer_college_2020.py b/python/evalio/datasets/newer_college_2020.py
index cf792d09..0cf65c56 100644
--- a/python/evalio/datasets/newer_college_2020.py
+++ b/python/evalio/datasets/newer_college_2020.py
@@ -179,7 +179,7 @@ def download(self):
             "parkland_mound": "1CMcmw9pAT1Mm-Zh-nS87i015CO-xFHwl",
         }[self.seq_name]
 
-        import gdown  # type: ignore
+        import gdown
 
         print(f"Downloading to {self.folder}...")
 
diff --git a/python/evalio/stats.py b/python/evalio/stats.py
index b3d3a256..b3ac10ca 100644
--- a/python/evalio/stats.py
+++ b/python/evalio/stats.py
@@ -327,7 +327,7 @@ def rte(
         # Compute deltas for all of ground truth poses
         dist = np.zeros(len(gt))
         for i in range(1, len(gt)):
-            diff: NDArray[np.float64] = gt.poses[i].trans - gt.poses[i - 1].trans  # type: ignore
+            diff: NDArray[np.float64] = gt.poses[i].trans - gt.poses[i - 1].trans
             dist[i] = np.sqrt(diff @ diff)
 
         cum_dist = np.cumsum(dist)
diff --git a/tests/test_io.py b/tests/test_io.py
index b6b45d07..aa00d74b 100644
--- a/tests/test_io.py
+++ b/tests/test_io.py
@@ -67,7 +67,7 @@ def test_trajectory_incremental_serde(tmp_path: Path):
 
     # must trigger entire rewrite to update metadata
     traj.open(path)
-    traj.metadata.sequence = "random_name"  # type: ignore
+    traj.metadata.sequence = "random_name"
     traj.rewrite()
     traj.close()
 
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index e1bcd9f3..ad1eb716 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -70,7 +70,7 @@ def default_params() -> dict[str, Param]:
     # bad ones
     ("unknown", pl.PipelineNotFound("unknown")),
     ({"pipeline": "unknown"}, pl.PipelineNotFound("unknown")),
-    ({"param1": 5}, pl.InvalidPipelineConfig("Need pipeline: {'param1': 5}")), # type: ignore
+    ({"param1": 5}, pl.InvalidPipelineConfig("Need pipeline: {'param1': 5}")), 
     ({"pipeline": "fake", "param3": 10}, pl.UnusedPipelineParam("param3", "fake")),
     ({"pipeline": "fake", "param1": "wrong_type"}, pl.InvalidPipelineParamType("param1", int, str)),
     ({"pipeline": "fake", "sweep": {"param1": [1.0, 2, 3]}}, pl.InvalidPipelineParamType("param1", int, float)),

From 0fb692430d2e594b7e36372f76637e814899e5ff Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Tue, 30 Sep 2025 22:29:54 -0400
Subject: [PATCH 14/40] Update stats command for more flexible window
 specifying

---
 pyproject.toml             |  1 +
 python/evalio/cli/stats.py | 79 +++++++++++++++-----------------------
 python/evalio/stats.py     | 73 ++++++++++++++++++++---------------
 3 files changed, 76 insertions(+), 77 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index f98758d2..d7929985 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -104,6 +104,7 @@ typeCheckingMode = "strict"
 stubPath = "python/typings"
 reportPrivateUsage = "none"
 reportConstantRedefinition = "none"
+reportUnnecessaryIsInstance = "none"
 
 [tool.bumpversion]
 allow_dirty = false
diff --git a/python/evalio/cli/stats.py b/python/evalio/cli/stats.py
index 0a737c40..b5fbaa3d 100644
--- a/python/evalio/cli/stats.py
+++ b/python/evalio/cli/stats.py
@@ -25,8 +25,7 @@
 def eval_dataset(
     dir: Path,
     visualize: bool,
-    window_kind: stats.WindowKind,
-    window_size: Optional[int | float],
+    windows: list[stats.WindowKind],
     metric: stats.MetricKind,
     length: Optional[int],
 ) -> Optional[list[dict[str, Any]]]:
@@ -95,18 +94,11 @@ def eval_dataset(
             gt_aligned.stamps = gt_aligned.stamps[:length]
             gt_aligned.poses = gt_aligned.poses[:length]
         ate = stats.ate(traj_aligned, gt_aligned).summarize(metric)
-        rte = stats.rte(traj_aligned, gt_aligned, window_kind, window_size).summarize(
-            metric
-        )
+        r.update({"ATEt": ate.trans, "ATEr": ate.rot})
 
-        r.update(
-            {
-                "RTEt": rte.trans,
-                "RTEr": rte.rot,
-                "ATEt": ate.trans,
-                "ATEr": ate.rot,
-            }
-        )
+        for w in windows:
+            rte = stats.rte(traj_aligned, gt_aligned, w).summarize(metric)
+            r.update({f"RTEt_{w.name()}": rte.trans, f"RTEr_{w.name()}": rte.rot})
 
         results.append(r)
 
@@ -126,11 +118,10 @@ def _contains_dir(directory: Path) -> bool:
 
 def evaluate(
     directories: list[Path],
-    window_size: Optional[float],
-    window_kind: stats.WindowKind,
+    windows: list[stats.WindowKind],
     metric: stats.MetricKind,
-    length: Optional[int],
-    visualize: bool,
+    length: Optional[int] = None,
+    visualize: bool = False,
 ) -> list[dict[str, Any]]:
     # Collect all bottom level directories
     bottom_level_dirs: list[Path] = []
@@ -144,8 +135,7 @@ def evaluate(
         delayed(eval_dataset)(
             d,
             visualize,
-            window_kind,
-            window_size,
+            windows,
             metric,
             length,
         )
@@ -166,14 +156,14 @@ def evaluate_typer(
     ] = False,
     # output options
     sort: Annotated[
-        str,
+        Optional[str],
         typer.Option(
             "-s",
             "--sort",
-            help="Sort results by the name of a column.",
+            help="Sort results by the name of a column. Defaults to RTEt.",
             rich_help_panel="Output options",
         ),
-    ] = "RTEt",
+    ] = None,
     reverse: Annotated[
         bool,
         typer.Option(
@@ -227,24 +217,22 @@ def evaluate_typer(
         ),
     ] = False,
     # metric options
-    window_size: Annotated[
-        Optional[float],
+    w_distance: Annotated[
+        Optional[list[float]],
         typer.Option(
-            "-w",
-            "--window-size",
-            help="Window size for RTE. Defaults to 100 time steps for time windows, 10 meters for distance windows.",
+            "--w-distance",
+            help="Window size in meters for RTE computation. May be repeated. Defaults to 30m.",
             rich_help_panel="Metric options",
         ),
     ] = None,
-    window_kind: Annotated[
-        stats.WindowKind,
+    w_scans: Annotated[
+        Optional[list[int]],
         typer.Option(
-            "-k",
-            "--window-kind",
-            help="Kind of window to use for RTE. Defaults to time.",
+            "--w-scans",
+            help="Window size in number of scans for RTE computation. May be repeated. Defaults to none.",
             rich_help_panel="Metric options",
         ),
-    ] = stats.WindowKind.time,
+    ] = None,
     metric: Annotated[
         stats.MetricKind,
         typer.Option(
@@ -291,26 +279,23 @@ def evaluate_typer(
     elif only_failed:
         filter_method = lambda r: original_filter(r) and r["status"] == "fail"  # noqa: E731
 
-    match window_kind:
-        case stats.WindowKind.distance:
-            if window_size is None:
-                window_size = 10
-            words = "meters"
-        case stats.WindowKind.time:
-            if window_size is None:
-                window_size = 200
-            words = "time steps"
+    windows: list[stats.WindowKind] = []
+    if w_scans is not None:
+        windows.extend([stats.ScanWindow(t) for t in w_scans])
+    if w_distance is not None:
+        windows.extend([stats.DistanceWindow(d) for d in w_distance])
+    if len(windows) == 0:
+        windows = [stats.DistanceWindow(30.0)]
+
+    if sort is None:
+        sort = f"RTEt_{windows[0].name()}"
 
     c = Console()
-    c.print(
-        f"Evaluating RTE over a window of {window_size} {words}, using metric {metric}."
-    )
 
     # ------------------------- Compute all results ------------------------- #
     results = evaluate(
         directories,
-        window_size,
-        window_kind,
+        windows,
         metric,
         length,
         visualize,
diff --git a/python/evalio/stats.py b/python/evalio/stats.py
index b3ac10ca..7dc0d309 100644
--- a/python/evalio/stats.py
+++ b/python/evalio/stats.py
@@ -8,7 +8,7 @@
 
 import numpy as np
 
-from typing import Optional, cast
+from typing import cast
 from numpy.typing import NDArray
 
 from copy import deepcopy
@@ -29,13 +29,31 @@ class MetricKind(StrEnum):
     """Sqrt of Sum of squared errors"""
 
 
-class WindowKind(StrEnum):
-    """Simple enum to define whether the window computed should be based on distance or time."""
+@dataclass
+class DistanceWindow:
+    """Dataclass to hold the parameters for a distance-based window."""
 
-    distance = auto()
-    """Window based on distance"""
-    time = auto()
-    """Window based on time"""
+    distance: float
+    """Distance in meters"""
+
+    def name(self) -> str:
+        """Get a string representation of the window."""
+        return f"{self.distance:.1f}m"
+
+
+@dataclass
+class ScanWindow:
+    """Dataclass to hold the parameters for a scan-based window."""
+
+    scans: int
+    """Number of scans"""
+
+    def name(self) -> str:
+        """Get a string representation of the window."""
+        return f"{self.scans}s"
+
+
+WindowKind = DistanceWindow | ScanWindow
 
 
 @dataclass(kw_only=True)
@@ -278,8 +296,7 @@ def ate(traj: Trajectory[M1], gt: Trajectory[M2]) -> Error:
 def rte(
     traj: Trajectory[M1],
     gt: Trajectory[M2],
-    kind: WindowKind = WindowKind.time,
-    window: Optional[float | int] = None,
+    window: WindowKind = DistanceWindow(30),
 ) -> Error:
     """Compute the Relative Trajectory Error (RTE) between two trajectories.
 
@@ -289,41 +306,35 @@ def rte(
     Args:
         traj (Trajectory): One of the trajectories
         gt (Trajectory): The other trajectory
-        kind (WindowKind, optional): The kind of window to use for the RTE. Defaults to WindowKind.time.
-        window (int | float, optional): Window size for the RTE. If window kind is distance, defaults to 10m. If time, defaults to 100 scans.
-
+        window (WindowKind, optional): The window to use for computing the RTE.
+            Either a [DistanceWindow][evalio.stats.DistanceWindow] or a [ScanWindow][evalio.stats.ScanWindow].
+            Defaults to DistanceWindow(30), which is a 30 meter window.
     Returns:
         Error: The computed error
     """
     if not _check_aligned(traj, gt):
         traj, gt = align(traj, gt)
 
-    if window is None:
-        match kind:
-            case WindowKind.distance:
-                window = 10
-            case WindowKind.time:
-                window = 200
-
-    if window <= 0:
+    if (isinstance(window, ScanWindow) and window.scans <= 0) or (
+        isinstance(window, DistanceWindow) and window.distance <= 0
+    ):
         raise ValueError("Window size must be positive")
 
-    if window > len(gt) - 1:
+    if isinstance(window, ScanWindow) and window.scans > len(gt) - 1:
         print_warning(f"Window size {window} is larger than number of poses {len(gt)}")
         return Error(rot=np.array([np.nan]), trans=np.array([np.nan]))
 
     window_deltas_poses: list[SE3] = []
     window_deltas_gts: list[SE3] = []
 
-    if kind == WindowKind.time:
-        assert isinstance(window, int), (
-            "Window size must be an integer for time-based RTE"
-        )
-        for i in range(len(gt) - window):
-            window_deltas_poses.append(traj.poses[i].inverse() * traj.poses[i + window])
-            window_deltas_gts.append(gt.poses[i].inverse() * gt.poses[i + window])
+    if isinstance(window, ScanWindow):
+        for i in range(len(gt) - window.scans):
+            window_deltas_poses.append(
+                traj.poses[i].inverse() * traj.poses[i + window.scans]
+            )
+            window_deltas_gts.append(gt.poses[i].inverse() * gt.poses[i + window.scans])
 
-    elif kind == WindowKind.distance:
+    elif isinstance(window, DistanceWindow):
         # Compute deltas for all of ground truth poses
         dist = np.zeros(len(gt))
         for i in range(1, len(gt)):
@@ -336,7 +347,9 @@ def rte(
 
         # Find our pairs for computation
         for i in range(len(gt)):
-            while end_idx < len(gt) and cum_dist[end_idx] - cum_dist[i] < window:
+            while (
+                end_idx < len(gt) and cum_dist[end_idx] - cum_dist[i] < window.distance
+            ):
                 end_idx += 1
 
             if end_idx >= len(gt):

From 87198ce2aa12f22c6768c3357fc0e8edbd08d47c Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Wed, 1 Oct 2025 10:28:57 -0400
Subject: [PATCH 15/40] Clean up stats window nomenclature

---
 python/evalio/cli/stats.py |  22 ++++----
 python/evalio/stats.py     | 101 ++++++++++++++++++++-----------------
 2 files changed, 67 insertions(+), 56 deletions(-)

diff --git a/python/evalio/cli/stats.py b/python/evalio/cli/stats.py
index b5fbaa3d..3a507bd6 100644
--- a/python/evalio/cli/stats.py
+++ b/python/evalio/cli/stats.py
@@ -217,19 +217,19 @@ def evaluate_typer(
         ),
     ] = False,
     # metric options
-    w_distance: Annotated[
+    w_meters: Annotated[
         Optional[list[float]],
         typer.Option(
-            "--w-distance",
+            "--w-meters",
             help="Window size in meters for RTE computation. May be repeated. Defaults to 30m.",
             rich_help_panel="Metric options",
         ),
     ] = None,
-    w_scans: Annotated[
-        Optional[list[int]],
+    w_seconds: Annotated[
+        Optional[list[float]],
         typer.Option(
-            "--w-scans",
-            help="Window size in number of scans for RTE computation. May be repeated. Defaults to none.",
+            "--w-seconds",
+            help="Window size in seconds for RTE computation. May be repeated. Defaults to none.",
             rich_help_panel="Metric options",
         ),
     ] = None,
@@ -280,12 +280,12 @@ def evaluate_typer(
         filter_method = lambda r: original_filter(r) and r["status"] == "fail"  # noqa: E731
 
     windows: list[stats.WindowKind] = []
-    if w_scans is not None:
-        windows.extend([stats.ScanWindow(t) for t in w_scans])
-    if w_distance is not None:
-        windows.extend([stats.DistanceWindow(d) for d in w_distance])
+    if w_seconds is not None:
+        windows.extend([stats.WindowSeconds(t) for t in w_seconds])
+    if w_meters is not None:
+        windows.extend([stats.WindowMeters(d) for d in w_meters])
     if len(windows) == 0:
-        windows = [stats.DistanceWindow(30.0)]
+        windows = [stats.WindowMeters(30.0)]
 
     if sort is None:
         sort = f"RTEt_{windows[0].name()}"
diff --git a/python/evalio/stats.py b/python/evalio/stats.py
index 7dc0d309..d8ee5c2f 100644
--- a/python/evalio/stats.py
+++ b/python/evalio/stats.py
@@ -2,7 +2,7 @@
 from typing_extensions import TypeVar
 
 from evalio.utils import print_warning
-from .types import Stamp, Trajectory, SE3, Metadata
+from . import types as ty
 
 from dataclasses import dataclass
 
@@ -14,7 +14,7 @@
 from copy import deepcopy
 
 
-def _check_overstep(stamps: list[Stamp], s: Stamp, idx: int) -> bool:
+def _check_overstep(stamps: list[ty.Stamp], s: ty.Stamp, idx: int) -> bool:
     return abs((stamps[idx - 1] - s).to_sec()) < abs((stamps[idx] - s).to_sec())
 
 
@@ -30,30 +30,30 @@ class MetricKind(StrEnum):
 
 
 @dataclass
-class DistanceWindow:
+class WindowMeters:
     """Dataclass to hold the parameters for a distance-based window."""
 
-    distance: float
+    value: float
     """Distance in meters"""
 
     def name(self) -> str:
         """Get a string representation of the window."""
-        return f"{self.distance:.1f}m"
+        return f"{self.value:.1f}m"
 
 
 @dataclass
-class ScanWindow:
-    """Dataclass to hold the parameters for a scan-based window."""
+class WindowSeconds:
+    """Dataclass to hold the parameters for a time-based window."""
 
-    scans: int
-    """Number of scans"""
+    value: float
+    """Duration of the window in seconds"""
 
     def name(self) -> str:
         """Get a string representation of the window."""
-        return f"{self.scans}s"
+        return f"{self.value}s"
 
 
-WindowKind = DistanceWindow | ScanWindow
+WindowKind = WindowMeters | WindowSeconds
 
 
 @dataclass(kw_only=True)
@@ -119,13 +119,13 @@ def median(self) -> Metric:
         )
 
 
-M1 = TypeVar("M1", bound=Metadata | None)
-M2 = TypeVar("M2", bound=Metadata | None)
+M1 = TypeVar("M1", bound=ty.Metadata | None)
+M2 = TypeVar("M2", bound=ty.Metadata | None)
 
 
 def align(
-    traj: Trajectory[M1], gt: Trajectory[M2], in_place: bool = False
-) -> tuple[Trajectory[M1], Trajectory[M2]]:
+    traj: ty.Trajectory[M1], gt: ty.Trajectory[M2], in_place: bool = False
+) -> tuple[ty.Trajectory[M1], ty.Trajectory[M2]]:
     """Align the trajectories both spatially and temporally.
 
     The resulting trajectories will be have the same origin as the second ("gt") trajectory.
@@ -146,7 +146,7 @@ def align(
     return traj, gt
 
 
-def align_poses(traj: Trajectory[M1], other: Trajectory[M2]):
+def align_poses(traj: ty.Trajectory[M1], other: ty.Trajectory[M2]):
     """Align the trajectory in place to another trajectory. Operates in place.
 
     This results in the current trajectory having an identical first pose to the other trajectory.
@@ -164,7 +164,7 @@ def align_poses(traj: Trajectory[M1], other: Trajectory[M2]):
         traj.poses[i] = delta * traj.poses[i]
 
 
-def align_stamps(traj1: Trajectory[M1], traj2: Trajectory[M2]):
+def align_stamps(traj1: ty.Trajectory[M1], traj2: ty.Trajectory[M2]):
     """Select the closest poses in traj1 and traj2. Operates in place.
 
     Does this by finding the higher frame rate trajectory and subsampling it to the closest poses of the other one.
@@ -203,8 +203,8 @@ def align_stamps(traj1: Trajectory[M1], traj2: Trajectory[M2]):
 
     # Align the two trajectories by subsampling keeping traj1 stamps
     traj1_idx = 0
-    traj1_stamps: list[Stamp] = []
-    traj1_poses: list[SE3] = []
+    traj1_stamps: list[ty.Stamp] = []
+    traj1_poses: list[ty.SE3] = []
     for i, stamp in enumerate(traj2.stamps):
         while traj1_idx < len(traj1) - 1 and traj1.stamps[traj1_idx] < stamp:
             traj1_idx += 1
@@ -228,7 +228,7 @@ def align_stamps(traj1: Trajectory[M1], traj2: Trajectory[M2]):
         traj1, traj2 = traj2, traj1  # type: ignore
 
 
-def _compute_metric(gts: list[SE3], poses: list[SE3]) -> Error:
+def _compute_metric(gts: list[ty.SE3], poses: list[ty.SE3]) -> Error:
     """Iterate and compute the SE(3) delta between two lists of poses.
 
     Args:
@@ -251,7 +251,7 @@ def _compute_metric(gts: list[SE3], poses: list[SE3]) -> Error:
     return Error(rot=error_r, trans=error_t)
 
 
-def _check_aligned(traj: Trajectory[M1], gt: Trajectory[M2]) -> bool:
+def _check_aligned(traj: ty.Trajectory[M1], gt: ty.Trajectory[M2]) -> bool:
     """Check if the two trajectories are aligned.
 
     This is done by checking if the first poses are identical, and if there's the same number of poses in both trajectories.
@@ -273,7 +273,7 @@ def _check_aligned(traj: Trajectory[M1], gt: Trajectory[M2]) -> bool:
     )
 
 
-def ate(traj: Trajectory[M1], gt: Trajectory[M2]) -> Error:
+def ate(traj: ty.Trajectory[M1], gt: ty.Trajectory[M2]) -> Error:
     """Compute the Absolute Trajectory Error (ATE) between two trajectories.
 
     Will check if the two trajectories are aligned and if not, will align them.
@@ -294,9 +294,9 @@ def ate(traj: Trajectory[M1], gt: Trajectory[M2]) -> Error:
 
 
 def rte(
-    traj: Trajectory[M1],
-    gt: Trajectory[M2],
-    window: WindowKind = DistanceWindow(30),
+    traj: ty.Trajectory[M1],
+    gt: ty.Trajectory[M2],
+    window: WindowKind = WindowMeters(30),
 ) -> Error:
     """Compute the Relative Trajectory Error (RTE) between two trajectories.
 
@@ -307,34 +307,38 @@ def rte(
         traj (Trajectory): One of the trajectories
         gt (Trajectory): The other trajectory
         window (WindowKind, optional): The window to use for computing the RTE.
-            Either a [DistanceWindow][evalio.stats.DistanceWindow] or a [ScanWindow][evalio.stats.ScanWindow].
-            Defaults to DistanceWindow(30), which is a 30 meter window.
+            Either a [WindowMeters][evalio.stats.WindowMeters] or a [WindowSeconds][evalio.stats.WindowSeconds].
+            Defaults to WindowMeters(30), which is a 30 meter window.
     Returns:
         Error: The computed error
     """
     if not _check_aligned(traj, gt):
         traj, gt = align(traj, gt)
 
-    if (isinstance(window, ScanWindow) and window.scans <= 0) or (
-        isinstance(window, DistanceWindow) and window.distance <= 0
-    ):
+    if window.value <= 0:
         raise ValueError("Window size must be positive")
 
-    if isinstance(window, ScanWindow) and window.scans > len(gt) - 1:
-        print_warning(f"Window size {window} is larger than number of poses {len(gt)}")
-        return Error(rot=np.array([np.nan]), trans=np.array([np.nan]))
+    window_deltas_poses: list[ty.SE3] = []
+    window_deltas_gts: list[ty.SE3] = []
 
-    window_deltas_poses: list[SE3] = []
-    window_deltas_gts: list[SE3] = []
+    if isinstance(window, WindowSeconds):
+        # Find our pairs for computation
+        end_idx = 1
+        duration = ty.Duration.from_sec(window.value)
 
-    if isinstance(window, ScanWindow):
-        for i in range(len(gt) - window.scans):
-            window_deltas_poses.append(
-                traj.poses[i].inverse() * traj.poses[i + window.scans]
-            )
-            window_deltas_gts.append(gt.poses[i].inverse() * gt.poses[i + window.scans])
+        for i in range(len(gt)):
+            while end_idx < len(gt) and gt.stamps[end_idx] - gt.stamps[i] < duration:
+                end_idx += 1
+
+            if end_idx >= len(gt):
+                break
 
-    elif isinstance(window, DistanceWindow):
+            window_deltas_poses.append(traj.poses[i].inverse() * traj.poses[end_idx])
+            window_deltas_gts.append(gt.poses[i].inverse() * gt.poses[end_idx])
+
+            end_idx_prev = end_idx
+
+    elif isinstance(window, WindowMeters):
         # Compute deltas for all of ground truth poses
         dist = np.zeros(len(gt))
         for i in range(1, len(gt)):
@@ -347,9 +351,7 @@ def rte(
 
         # Find our pairs for computation
         for i in range(len(gt)):
-            while (
-                end_idx < len(gt) and cum_dist[end_idx] - cum_dist[i] < window.distance
-            ):
+            while end_idx < len(gt) and cum_dist[end_idx] - cum_dist[i] < window.value:
                 end_idx += 1
 
             if end_idx >= len(gt):
@@ -362,5 +364,14 @@ def rte(
 
             end_idx_prev = end_idx
 
+    if len(window_deltas_poses) == 0:
+        if isinstance(traj.metadata, ty.Experiment):
+            print_warning(
+                f"No windows found with size {window} for '{traj.metadata.name}' on '{traj.metadata.sequence}'"
+            )
+        else:
+            print_warning(f"No windows found with size {window}")
+        return Error(rot=np.array([np.nan]), trans=np.array([np.nan]))
+
     # Compute the RTE
     return _compute_metric(window_deltas_gts, window_deltas_poses)

From e36b05b4c3473797fd7e9ac0ebb13471ab699381 Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Wed, 1 Oct 2025 11:14:51 -0400
Subject: [PATCH 16/40] Add EVALIO_CUSTOM hooks

---
 docs/quickstart.md            | 13 +++++++------
 python/evalio/__init__.py     | 34 ++++++++++++++++++++++++++++++++++
 python/evalio/cli/__init__.py | 26 ++++++++++++++++++++++++--
 python/evalio/cli/stats.py    |  3 +--
 4 files changed, 66 insertions(+), 10 deletions(-)

diff --git a/docs/quickstart.md b/docs/quickstart.md
index 22b4e678..2142bb20 100644
--- a/docs/quickstart.md
+++ b/docs/quickstart.md
@@ -31,18 +31,18 @@ evalio downloads data to the path given by `-D`, `EVALIO_DATA` environment varia
 
 Once downloaded, a trajectory can then be easily used in python,
 ```python
-from evalio.datasets import Hilti2022
+from evalio import datasets as ds
 
 # for all data
-for mm in Hilti2022.basement_2:
+for mm in ds.Hilti2022.basement_2:
     print(mm)
 
 # for lidars
-for scan in Hilti2022.basement_2.lidar():
+for scan in ds.Hilti2022.basement_2.lidar():
     print(scan)
 
 # for imu
-for imu in Hilti2022.basement_2.imu():
+for imu in ds.Hilti2022.basement_2.imu():
     print(imu)
 ```
 
@@ -52,7 +52,7 @@ import matplotlib.pyplot as plt
 import numpy as np
 
 # get the 10th scan
-scan = Hilti2022.basement_2.get_one_lidar(10)
+scan = ds.Hilti2022.basement_2.get_one_lidar(10)
 # always in row-major order, with stamp at start of scan
 x = np.array([p.x for p in scan.points])
 y = np.array([p.y for p in scan.points])
@@ -68,7 +68,7 @@ from evalio.rerun import convert
 
 rr.init("evalio")
 rr.connect_tcp()
-for scan in Hilti2022.basement_2.lidar():
+for scan in ds.Hilti2022.basement_2.lidar():
     rr.set_time("timeline", timestamp=scan.stamp.to_sec())
     rr.log("lidar", convert(scan, color=[255, 0, 255]))
 ```
@@ -101,6 +101,7 @@ evalio stats results
 
 More complex experiments can be run, including varying pipeline parameters, via specifying a config file,
 ```yaml
+# If not specified, defaults to ./evalio_results/config_file_name
 output_dir: ./results/
 
 datasets:
diff --git a/python/evalio/__init__.py b/python/evalio/__init__.py
index 3acdb163..612bb560 100644
--- a/python/evalio/__init__.py
+++ b/python/evalio/__init__.py
@@ -1,5 +1,8 @@
 import atexit
+from typing import cast
 import warnings
+import os
+import importlib
 
 from tqdm import TqdmExperimentalWarning
 
@@ -21,6 +24,37 @@ def cleanup():
 # Ignore tqdm rich warnings
 warnings.filterwarnings("ignore", category=TqdmExperimentalWarning)
 
+
+# Register any custom modules specified in the environment
+def _register_custom_modules(module_name: str):
+    # Make sure we only attempt to register each module once
+    if not hasattr(_register_custom_modules, "attempted"):
+        _register_custom_modules.attempted = set()  # type: ignore
+
+    if module_name in _register_custom_modules.attempted:  # type: ignore
+        return
+    _register_custom_modules.attempted.add(module_name)  # type: ignore
+
+    try:
+        module = importlib.import_module(module_name)
+        pl_out = pipelines.register_pipeline(module=module)
+        ds_out = datasets.register_dataset(module=module)
+
+        if cast(int, pl_out) + cast(int, ds_out) == 0:
+            utils.print_warning(
+                f"No pipelines or datasets found in custom module '{module_name}'"
+            )
+
+    except ImportError:
+        utils.print_warning(f"Failed to import custom module '{module_name}'")
+
+
+if "EVALIO_CUSTOM" in os.environ:
+    for module_name in os.environ["EVALIO_CUSTOM"].split(","):
+        module_name = module_name.strip()
+        _register_custom_modules(module_name)
+
+
 __version__ = "0.3.0"
 __all__ = [
     "_abi_tag",
diff --git a/python/evalio/cli/__init__.py b/python/evalio/cli/__init__.py
index 1d6cf95e..22a16e49 100644
--- a/python/evalio/cli/__init__.py
+++ b/python/evalio/cli/__init__.py
@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Annotated, Optional
+from typing import Annotated, Any, Optional
 
 import typer
 
@@ -38,10 +38,21 @@ def data_callback(value: Optional[Path]):
     """
     Set the data directory.
     """
-    if value:
+    if value is not None:
         set_data_dir(value)
 
 
+def module_callback(value: Optional[list[str]]) -> list[Any]:
+    """
+    Set the module to use.
+    """
+    if value is not None:
+        for module in value:
+            evalio._register_custom_modules(module)
+
+    return []
+
+
 @app.callback()
 def global_options(
     # Marking this as a str for now to get autocomplete to work,
@@ -58,6 +69,17 @@ def global_options(
             callback=data_callback,
         ),
     ] = None,
+    custom_modules: Annotated[
+        Optional[list[str]],
+        typer.Option(
+            "-M",
+            "--module",
+            help="Custom module to load (for custom datasets or pipelines). Can be used multiple times.",
+            show_default=False,
+            rich_help_panel="Global options",
+            callback=module_callback,
+        ),
+    ] = None,
     version: Annotated[
         bool,
         typer.Option(
diff --git a/python/evalio/cli/stats.py b/python/evalio/cli/stats.py
index 3a507bd6..032842a4 100644
--- a/python/evalio/cli/stats.py
+++ b/python/evalio/cli/stats.py
@@ -9,8 +9,7 @@
 from rich.console import Console
 from rich import box
 
-from evalio import types as ty
-from evalio import stats
+from evalio import types as ty, stats
 
 import numpy as np
 import typer

From e5669e96435a486e82d6ad30798677b6ade90e9e Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Wed, 1 Oct 2025 14:50:24 -0400
Subject: [PATCH 17/40] Update all documentation of rewrite

---
 cpp/bindings/types.h               |  23 ++++---
 docs/ref/datasets.md               |  21 +-----
 mkdocs.yml                         |  12 ++--
 python/evalio/cli/ls.py            |   6 +-
 python/evalio/datasets/__init__.py |  29 ++++----
 python/evalio/datasets/base.py     |  62 ++++++++---------
 python/evalio/datasets/parser.py   |  39 +++++++++++
 python/evalio/pipelines/parser.py  |  16 ++---
 python/evalio/rerun.py             |   2 +-
 python/evalio/stats.py             |  11 +--
 python/evalio/types/__init__.py    |   3 +-
 python/evalio/types/base.py        | 106 +++++++++++++++++++++++------
 python/evalio/types/extended.py    |  22 ++++--
 python/evalio/utils.py             |   2 +-
 14 files changed, 228 insertions(+), 126 deletions(-)

diff --git a/cpp/bindings/types.h b/cpp/bindings/types.h
index 95ed0260..285777cf 100644
--- a/cpp/bindings/types.h
+++ b/cpp/bindings/types.h
@@ -68,8 +68,8 @@ inline void makeTypes(nb::module_& m) {
       }
     )
     .doc() =
-    "Duration class for representing a positive or negative delta time, uses "
-    "int64 as the underlying data storage for nanoseconds.";
+    "Duration class for representing a positive or negative delta time. \n\n"
+    "Uses int64 as the underlying data storage for nanoseconds.";
 
   nb::class_<Stamp>(m, "Stamp")
     .def(
@@ -133,8 +133,8 @@ inline void makeTypes(nb::module_& m) {
       }
     )
     .doc() =
-    "Stamp class for representing an absolute point in time, uses uint32 as "
-    "the underlying data storage for seconds and nanoseconds.";
+    "Stamp class for representing an absolute point in time.\n\n"
+    "Uses uint32 as the underlying data storage for seconds and nanoseconds.";
   ;
 
   // Lidar
@@ -232,7 +232,7 @@ inline void makeTypes(nb::module_& m) {
       }
     )
     .doc() =
-    "Point is a general point structure in evalio, with common "
+    "Point is the general point structure in evalio, with common "
     "point cloud attributes included.";
 
   nb::class_<LidarMeasurement>(m, "LidarMeasurement")
@@ -284,8 +284,9 @@ inline void makeTypes(nb::module_& m) {
     )
     .doc() =
     "LidarMeasurement is a structure for storing a point cloud "
-    "measurement, with a timestamp and a vector of points. Note, "
-    "the stamp always represents the _start_ of the scan. "
+    "measurement, with a timestamp and a vector of points.\n\n"
+
+    "Note, the stamp always represents the _start_ of the scan. "
     "Additionally, the points are always in row major format.";
 
   nb::class_<LidarParams>(m, "LidarParams")
@@ -522,7 +523,7 @@ inline void makeTypes(nb::module_& m) {
       }
     )
     .doc() =
-    "SO3 class for representing a 3D rotation using a quaternion. "
+    "SO3 class for representing a 3D rotation using a quaternion.\n\n"
     "This is outfitted with some basic functionality, but mostly "
     "intended for storage and converting between types.";
 
@@ -599,9 +600,9 @@ inline void makeTypes(nb::module_& m) {
     )
     .doc() =
     "SE3 class for representing a 3D rigid body transformation "
-    "using a quaternion and a translation vector. This is outfitted "
-    "with some basic functionality, but mostly intended for storage "
-    "and converting between types.";
+    "using a quaternion and a translation vector.\n\n"
+    "This is outfitted with some basic functionality, but is mostly "
+    "intended for storage and converting between types.";
 }
 
 } // namespace evalio
diff --git a/docs/ref/datasets.md b/docs/ref/datasets.md
index 28506a51..5725ed9c 100644
--- a/docs/ref/datasets.md
+++ b/docs/ref/datasets.md
@@ -2,23 +2,4 @@ For more information about the datasets included in evalio, see the [included da
 
 ::: evalio.datasets
     options:
-        show_submodules: true
-        members:
-            - Dataset
-            - DatasetIterator
-            - RosbagIter
-            - RawDataIter
-            - get_data_dir
-            - set_data_dir
-
-::: evalio.datasets
-    options:
-        show_root_toc_entry: false
-        show_labels: false
-        filters:
-            - "!Dataset"
-            - "!DatasetIterator"
-            - "!RosbagIter"
-            - "!RawDataIter"
-            - "!get_data_dir"
-            - "!set_data_dir"
\ No newline at end of file
+        members_order: source
\ No newline at end of file
diff --git a/mkdocs.yml b/mkdocs.yml
index 4f5bc656..82b2542a 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -69,7 +69,7 @@ plugins:
   search:
     lang: en
 
-  # autogenerate evalio ls pages
+  # autogenerate evalio ls & cli pages
   gen-files:
     scripts:
       - docs/included.py
@@ -81,9 +81,7 @@ plugins:
     handlers:
       python:
         options:
-          # TODO: May want to remove this once all docs are added
-          # show_if_no_docstring: true # show everything
-          show_source: false # don't show source code
+          show_source: true # don't show source code
           separate_signature: true # show the signature in a separate section
           show_signature_annotations: true # include types
           signature_crossrefs: true # show cross-references in the signature
@@ -95,6 +93,11 @@ plugins:
           # show_labels: false
           # show_category_heading: true
           docstring_style: google
+          summary:
+            attributes: true
+            functions: true
+            classes: true
+            modules: false
 
 markdown_extensions:
   # misc
@@ -122,6 +125,7 @@ extra_javascript:
   # https://squidfunk.github.io/mkdocs-material/reference/data-tables/#sortable-tables-docsjavascriptstablesortjs
   - javascripts/tablesort.js
   - https://unpkg.com/tablesort@5.3.0/dist/tablesort.min.js
+  # latex
   - javascripts/katex.js
   - https://unpkg.com/katex@0/dist/katex.min.js
   - https://unpkg.com/katex@0/dist/contrib/auto-render.min.js
diff --git a/python/evalio/cli/ls.py b/python/evalio/cli/ls.py
index a7fb04a3..8d60c1dc 100644
--- a/python/evalio/cli/ls.py
+++ b/python/evalio/cli/ls.py
@@ -14,11 +14,11 @@
 T = TypeVar("T")
 
 
-def unique(lst: list[T]):
+def unique(lst: list[T]) -> list[T]:
     """Get unique elements from a list while preserving order
 
     Returns:
-        _type_: Unique list
+        List of unique elements
     """
     return list(dict.fromkeys(lst))
 
@@ -30,7 +30,7 @@ def extract_len(d: ds.Dataset) -> str:
         d (Dataset): Dataset to get length of
 
     Returns:
-        str: Length of dataset
+        Length of dataset in minutes or '-' if length is unknown
     """
     length = d.quick_len()
     if length is None:
diff --git a/python/evalio/datasets/__init__.py b/python/evalio/datasets/__init__.py
index 5e762e88..f9db5a66 100644
--- a/python/evalio/datasets/__init__.py
+++ b/python/evalio/datasets/__init__.py
@@ -1,10 +1,11 @@
 from .base import Dataset, DatasetIterator, get_data_dir, set_data_dir
+from .loaders import RawDataIter, RosbagIter
+
 from .botanic_garden import BotanicGarden
 from .cumulti import CUMulti
 from .enwide import EnWide
 from .helipr import HeLiPR
 from .hilti_2022 import Hilti2022
-from .loaders import RawDataIter, RosbagIter
 from .multi_campus import MultiCampus
 from .newer_college_2020 import NewerCollege2020
 from .newer_college_2021 import NewerCollege2021
@@ -25,21 +26,15 @@
 )
 
 __all__ = [
-    "get_data_dir",
-    "set_data_dir",
+    # base imports
     "Dataset",
     "DatasetIterator",
-    "BotanicGarden",
-    "CUMulti",
-    "EnWide",
-    "HeLiPR",
-    "Hilti2022",
-    "NewerCollege2020",
-    "NewerCollege2021",
-    "MultiCampus",
-    "OxfordSpires",
+    "get_data_dir",
+    "set_data_dir",
+    # loaders
     "RawDataIter",
     "RosbagIter",
+    # parser
     "all_datasets",
     "get_dataset",
     "all_sequences",
@@ -51,4 +46,14 @@
     "InvalidDatasetConfig",
     "DatasetConfigError",
     "DatasetConfig",
+    # datasets
+    "BotanicGarden",
+    "CUMulti",
+    "EnWide",
+    "HeLiPR",
+    "Hilti2022",
+    "NewerCollege2020",
+    "NewerCollege2021",
+    "MultiCampus",
+    "OxfordSpires",
 ]
diff --git a/python/evalio/datasets/base.py b/python/evalio/datasets/base.py
index dcf5d36e..157aed38 100644
--- a/python/evalio/datasets/base.py
+++ b/python/evalio/datasets/base.py
@@ -35,7 +35,7 @@ def imu_iter(self) -> Iterator[ImuMeasurement]:
         """Main interface for iterating over IMU measurements.
 
         Yields:
-            Iterator[ImuMeasurement]: Iterator of IMU measurements.
+            Iterator of IMU measurements.
         """
         ...
 
@@ -43,7 +43,7 @@ def lidar_iter(self) -> Iterator[LidarMeasurement]:
         """Main interface for iterating over Lidar measurements.
 
         Yields:
-            Iterator[LidarMeasurement]: Iterator of Lidar measurements.
+            Iterator of Lidar measurements.
         """
         ...
 
@@ -51,7 +51,7 @@ def __iter__(self) -> Iterator[Measurement]:
         """Main interface for iterating over all measurements.
 
         Yields:
-            Iterator[Measurement]: Iterator of all measurements (IMU and Lidar).
+            Iterator of all measurements (IMU and Lidar).
         """
 
         ...
@@ -61,7 +61,7 @@ def __len__(self) -> int:
         """Number of lidar scans.
 
         Returns:
-            int: Number of lidar scans.
+            Number of lidar scans.
         """
         ...
 
@@ -79,7 +79,7 @@ def data_iter(self) -> DatasetIterator:
         Provides an iterator over the dataset's measurements.
 
         Returns:
-            DatasetIterator: An iterator that yields measurements from the dataset.
+            An iterator that yields measurements from the dataset.
         """
         ...
 
@@ -89,7 +89,7 @@ def ground_truth_raw(self) -> Trajectory:
         Retrieves the raw ground truth trajectory, as represented in the ground truth frame.
 
         Returns:
-            Trajectory: The raw ground truth trajectory data.
+            The raw ground truth trajectory data.
         """
         ...
 
@@ -98,7 +98,7 @@ def imu_T_lidar(self) -> SE3:
         """Returns the transformation from IMU to Lidar frame.
 
         Returns:
-            SE3: Transformation from IMU to Lidar frame.
+            Transformation from IMU to Lidar frame.
         """
         ...
 
@@ -106,7 +106,7 @@ def imu_T_gt(self) -> SE3:
         """Retrieves the transformation from IMU to ground truth frame.
 
         Returns:
-            SE3: Transformation from IMU to ground truth frame.
+            Transformation from IMU to ground truth frame.
         """
         ...
 
@@ -114,7 +114,7 @@ def imu_params(self) -> ImuParams:
         """Specifies the parameters of the IMU.
 
         Returns:
-            ImuParams: Parameters of the IMU.
+            Parameters of the IMU.
         """
         ...
 
@@ -122,7 +122,7 @@ def lidar_params(self) -> LidarParams:
         """Specifies the parameters of the Lidar.
 
         Returns:
-            LidarParams: Parameters of the Lidar.
+            Parameters of the Lidar.
         """
         ...
 
@@ -132,7 +132,7 @@ def files(self) -> Sequence[str | Path]:
         If a returned type is a Path, it will be checked as is. If it is a string, it will be prepended with [folder][evalio.datasets.Dataset.folder].
 
         Returns:
-            list[str]: _description_
+            List of files required to run this dataset.
         """
         ...
 
@@ -142,7 +142,7 @@ def url() -> str:
         """Webpage with the dataset information.
 
         Returns:
-            str: URL of the dataset webpage.
+            URL of the dataset webpage.
         """
         return "-"
 
@@ -150,7 +150,7 @@ def environment(self) -> str:
         """Environment where the dataset was collected.
 
         Returns:
-            str: Environment where the dataset was collected.
+            Environment where the dataset was collected.
         """
         return "-"
 
@@ -158,7 +158,7 @@ def vehicle(self) -> str:
         """Vehicle used to collect the dataset.
 
         Returns:
-            str: Vehicle used to collect the dataset.
+            Vehicle used to collect the dataset.
         """
         return "-"
 
@@ -166,7 +166,7 @@ def quick_len(self) -> Optional[int]:
         """Hardcoded number of lidar scans in the dataset, rather than computing by loading all the data (slow).
 
         Returns:
-            Optional[int]: Number of lidar scans in the dataset. None if not available.
+            Number of lidar scans in the dataset. None if not available.
         """
         return None
 
@@ -188,7 +188,7 @@ def dataset_name(cls) -> str:
         This is the name that will be used when parsing directly from a string. Currently is automatically generated from the class name, but can be overridden.
 
         Returns:
-            str: _description_
+            Name of the dataset.
         """
         return pascal_to_snake(cls.__name__)
 
@@ -197,7 +197,7 @@ def is_downloaded(self) -> bool:
         """Verify if the dataset is downloaded.
 
         Returns:
-            bool: True if the dataset is downloaded, False otherwise.
+            True if the dataset is downloaded, False otherwise.
         """
         self._warn_default_dir()
         for f in self.files():
@@ -214,7 +214,7 @@ def ground_truth(self) -> Trajectory[GroundTruth]:
         """Get the ground truth trajectory in the **IMU** frame, rather than the ground truth frame as returned in [ground_truth_raw][evalio.datasets.Dataset.ground_truth_raw].
 
         Returns:
-            Trajectory: The ground truth trajectory in the IMU frame.
+            The ground truth trajectory in the IMU frame.
         """
         gt_traj = self.ground_truth_raw()
         gt_T_imu = self.imu_T_gt().inverse()
@@ -253,7 +253,7 @@ def __len__(self) -> int:
         If quick_len is available, it will be used. Otherwise, it will load the entire dataset to get the length.
 
         Returns:
-            int: Number of lidar scans.
+            Number of lidar scans.
         """
         if (length := self.quick_len()) is not None:
             return length
@@ -265,7 +265,7 @@ def __iter__(self) -> Iterator[Measurement]:  # type: ignore
         """Main interface for iterating over measurements of all types.
 
         Returns:
-            Iterator[Measurement]: Iterator of all measurements (IMU and Lidar).
+            Iterator of all measurements (IMU and Lidar).
         """
         self._fail_not_downloaded()
         return self.data_iter().__iter__()
@@ -274,7 +274,7 @@ def imu(self) -> Iterable[ImuMeasurement]:
         """Iterate over just IMU measurements.
 
         Returns:
-            Iterable[ImuMeasurement]: Iterator of IMU measurements.
+            Iterator of IMU measurements.
         """
         self._fail_not_downloaded()
         return self.data_iter().imu_iter()
@@ -283,7 +283,7 @@ def lidar(self) -> Iterable[LidarMeasurement]:
         """Iterate over just Lidar measurements.
 
         Returns:
-            Iterable[LidarMeasurement]: Iterator of Lidar measurements.
+            Iterator of Lidar measurements.
         """
         self._fail_not_downloaded()
         return self.data_iter().lidar_iter()
@@ -297,7 +297,7 @@ def get_one_lidar(self, idx: int = 0) -> LidarMeasurement:
             idx (int, optional): Index of measurement to get. Defaults to 0.
 
         Returns:
-            LidarMeasurement: The Lidar measurement at the given index.
+            The Lidar measurement at the given index.
         """
         return next(islice(self.lidar(), idx, idx + 1))
 
@@ -310,7 +310,7 @@ def get_one_imu(self, idx: int = 0) -> ImuMeasurement:
             idx (int, optional): Index of measurement to get. Defaults to 0.
 
         Returns:
-            ImuMeasurement: The IMU measurement at the given index.
+            The IMU measurement at the given index.
         """
         return next(islice(self.imu(), idx, idx + 1))
 
@@ -323,7 +323,7 @@ def seq_name(self) -> str:
         """Name of the sequence, in snake case.
 
         Returns:
-            str: Name of the sequence.
+            Name of the sequence.
         """
         return self.value
 
@@ -334,7 +334,7 @@ def full_name(self) -> str:
         Example: "dataset_name/sequence_name"
 
         Returns:
-            str: Full name of the dataset.
+            Full name of the dataset.
         """
         return f"{self.dataset_name()}/{self.seq_name}"
 
@@ -343,7 +343,7 @@ def sequences(cls) -> list["Dataset"]:
         """All sequences in the dataset.
 
         Returns:
-            list[Dataset]: List of all sequences in the dataset.
+            List of all sequences in the dataset.
         """
         return list(cls.__members__.values())
 
@@ -352,7 +352,7 @@ def folder(self) -> Path:
         """The folder in the global dataset directory where this dataset is stored.
 
         Returns:
-            Path: Path to the dataset folder.
+            Path to the dataset folder.
         """
         global _DATA_DIR
         return _DATA_DIR / self.full_name
@@ -361,7 +361,7 @@ def size_on_disk(self) -> Optional[float]:
         """Shows the size of the dataset on disk, in GB.
 
         Returns:
-            Optional[float]: Size of the dataset on disk, in GB. None if the dataset is not downloaded.
+            Size of the dataset on disk, in GB. None if the dataset is not downloaded.
         """
 
         if not self.is_downloaded():
@@ -383,10 +383,10 @@ def set_data_dir(directory: Path):
 
 
 def get_data_dir() -> Path:
-    """Get the global data directory. This will be used to store the downloaded data.
+    """Get the global data directory. This is where downloaded data is stored.
 
     Returns:
-        Path: Directory where datasets are stored.
+        Directory where datasets are stored.
     """
     global _DATA_DIR
     return _DATA_DIR
diff --git a/python/evalio/datasets/parser.py b/python/evalio/datasets/parser.py
index 9c3e66f6..ee73af15 100644
--- a/python/evalio/datasets/parser.py
+++ b/python/evalio/datasets/parser.py
@@ -12,12 +12,16 @@
 
 
 class DatasetNotFound(CustomException):
+    """Exception raised when a dataset is not found."""
+
     def __init__(self, name: str):
         super().__init__(f"Dataset '{name}' not found")
         self.name = name
 
 
 class SequenceNotFound(CustomException):
+    """Exception raised when a sequence is not found."""
+
     def __init__(self, name: str):
         super().__init__(f"Sequence '{name}' not found")
         self.name = name
@@ -47,6 +51,15 @@ def register_dataset(
     dataset: Optional[type[Dataset]] = None,
     module: Optional[ModuleType | str] = None,
 ) -> int | ImportError:
+    """Register a dataset.
+
+    Args:
+        dataset (Optional[type[Dataset]], optional): The dataset class to register. Defaults to None.
+        module (Optional[ModuleType  |  str], optional): The module containing datasets to register. Defaults to None.
+
+    Returns:
+        The number of datasets registered or an ImportError.
+    """
     global _DATASETS
 
     total = 0
@@ -69,21 +82,47 @@ def register_dataset(
 
 
 def all_datasets() -> dict[str, type[Dataset]]:
+    """Get all registered datasets.
+
+    Returns:
+        A dictionary mapping dataset names to their classes.
+    """
     global _DATASETS
     return {d.dataset_name(): d for d in _DATASETS}
 
 
 def get_dataset(name: str) -> type[Dataset] | DatasetNotFound:
+    """Get a registered dataset by name.
+
+    Args:
+        name (str): The name of the dataset to retrieve.
+
+    Returns:
+        The dataset class if found, or a DatasetNotFound error.
+    """
     return all_datasets().get(name, DatasetNotFound(name))
 
 
 def all_sequences() -> dict[str, Dataset]:
+    """Get all sequences from all registered datasets.
+
+    Returns:
+        A dictionary mapping sequence names to their dataset classes.
+    """
     return {
         seq.full_name: seq for d in all_datasets().values() for seq in d.sequences()
     }
 
 
 def get_sequence(name: str) -> Dataset | SequenceNotFound:
+    """Get a registered sequence by name.
+
+    Args:
+        name (str): The name of the sequence to retrieve.
+
+    Returns:
+        The dataset object if found, or a SequenceNotFound error.
+    """
     return all_sequences().get(name, SequenceNotFound(name))
 
 
diff --git a/python/evalio/pipelines/parser.py b/python/evalio/pipelines/parser.py
index 76d86bd1..93d96bdc 100644
--- a/python/evalio/pipelines/parser.py
+++ b/python/evalio/pipelines/parser.py
@@ -99,7 +99,7 @@ def all_pipelines() -> dict[str, type[Pipeline]]:
     """Get all registered pipelines.
 
     Returns:
-        dict[str, type[Pipeline]]: A dictionary mapping pipeline names to their classes.
+        A dictionary mapping pipeline names to their classes.
     """
     global _PIPELINES
     return {p.name(): p for p in _PIPELINES}
@@ -112,7 +112,7 @@ def get_pipeline(name: str) -> type[Pipeline] | PipelineNotFound:
         name (str): The name of the pipeline.
 
     Returns:
-        Optional[type[Pipeline]]: The pipeline class, or None if not found.
+        The pipeline class if found, otherwise a PipelineNotFound error.
     """
     return all_pipelines().get(name, PipelineNotFound(name))
 
@@ -145,7 +145,7 @@ def _sweep(
 def validate_params(
     pipe: type[Pipeline],
     params: dict[str, Param],
-) -> None | PipelineConfigError:
+) -> None | InvalidPipelineParamType | UnusedPipelineParam:
     """Validate the parameters for a given pipeline.
 
     Args:
@@ -153,7 +153,7 @@ def validate_params(
         params (dict[str, Param]): The parameters to validate.
 
     Returns:
-        Optional[PipelineConfigError]: An error if validation fails, otherwise None.
+        An error if validation fails, otherwise None.
     """
     default_params = pipe.default_params()
     for p in params:
@@ -171,14 +171,6 @@ def validate_params(
 def parse_config(
     p: str | dict[str, Param] | Sequence[str | dict[str, Param]],
 ) -> list[tuple[str, type[Pipeline], dict[str, Param]]] | PipelineConfigError:
-    """Parse a pipeline configuration.
-
-    Args:
-        p (str | dict[str, Param] | Sequence[str | dict[str, Param]]): The pipeline configuration.
-
-    Returns:
-        list[tuple[type[Pipeline], dict[str, Param]]]: A list of tuples containing the pipeline class and its parameters.
-    """
     if isinstance(p, str):
         pipe = get_pipeline(p)
         if isinstance(pipe, PipelineNotFound):
diff --git a/python/evalio/rerun.py b/python/evalio/rerun.py
index b4b6d2f5..798efd3f 100644
--- a/python/evalio/rerun.py
+++ b/python/evalio/rerun.py
@@ -383,7 +383,7 @@ def convert(
             ValueError: If the object is not an implemented type for conversion.
 
         Returns:
-            rr.Transform3D | rr.Points3D: Rerun type.
+            Rerun type.
         """
         # If we have an empty list, assume it's a point cloud with no points
         if isinstance(obj, list) and len(obj) == 0:  # type: ignore
diff --git a/python/evalio/stats.py b/python/evalio/stats.py
index d8ee5c2f..1b7a4ba7 100644
--- a/python/evalio/stats.py
+++ b/python/evalio/stats.py
@@ -54,6 +54,7 @@ def name(self) -> str:
 
 
 WindowKind = WindowMeters | WindowSeconds
+"""Type alias for either a [WindowMeters][evalio.stats.WindowMeters] or a [WindowSeconds][evalio.stats.WindowSeconds]."""
 
 
 @dataclass(kw_only=True)
@@ -89,7 +90,7 @@ def summarize(self, metric: MetricKind) -> Metric:
                 either mean, median, or sse.
 
         Returns:
-            Metric: The summarized error
+            The summarized error
         """
         match metric:
             case MetricKind.mean:
@@ -236,7 +237,7 @@ def _compute_metric(gts: list[ty.SE3], poses: list[ty.SE3]) -> Error:
         poses (list[SE3]): The other list of poses
 
     Returns:
-        Error: The computed error
+        The computed error
     """
     assert len(gts) == len(poses)
 
@@ -261,7 +262,7 @@ def _check_aligned(traj: ty.Trajectory[M1], gt: ty.Trajectory[M2]) -> bool:
         gt (Trajectory): The other trajectory
 
     Returns:
-        bool: True if the two trajectories are aligned, False otherwise
+        True if the two trajectories are aligned, False otherwise
     """
     # Check if the two trajectories are aligned
     delta = gt.poses[0].inverse() * traj.poses[0]
@@ -284,7 +285,7 @@ def ate(traj: ty.Trajectory[M1], gt: ty.Trajectory[M2]) -> Error:
         gt (Trajectory): The other trajectory
 
     Returns:
-        Error: The computed error
+        The computed error
     """
     if not _check_aligned(traj, gt):
         traj, gt = align(traj, gt)
@@ -310,7 +311,7 @@ def rte(
             Either a [WindowMeters][evalio.stats.WindowMeters] or a [WindowSeconds][evalio.stats.WindowSeconds].
             Defaults to WindowMeters(30), which is a 30 meter window.
     Returns:
-        Error: The computed error
+        The computed error
     """
     if not _check_aligned(traj, gt):
         traj, gt = align(traj, gt)
diff --git a/python/evalio/types/__init__.py b/python/evalio/types/__init__.py
index 9389a0cd..8612fc73 100644
--- a/python/evalio/types/__init__.py
+++ b/python/evalio/types/__init__.py
@@ -10,7 +10,7 @@
     Stamp,
 )
 
-from .base import Param, Trajectory, Metadata, GroundTruth
+from .base import Param, Trajectory, Metadata, GroundTruth, FailedMetadataParse
 from .extended import Experiment, ExperimentStatus
 
 
@@ -27,6 +27,7 @@
     "Stamp",
     # base includes
     "GroundTruth",
+    "FailedMetadataParse",
     "Metadata",
     "Param",
     "Trajectory",
diff --git a/python/evalio/types/base.py b/python/evalio/types/base.py
index 2d49e0e4..c7cdc0fa 100644
--- a/python/evalio/types/base.py
+++ b/python/evalio/types/base.py
@@ -9,7 +9,6 @@
 from dataclasses import asdict, dataclass, field
 import csv
 from _csv import Writer
-from enum import Enum
 from io import TextIOWrapper
 from typing_extensions import TypeVar
 from evalio.utils import print_warning
@@ -17,7 +16,7 @@
 import yaml
 
 from pathlib import Path
-from typing import Any, ClassVar, Generic, Optional, Self, cast
+from typing import Any, ClassVar, Generic, Iterator, Optional, Self, cast
 
 from evalio._cpp.types import (  # type: ignore
     SE3,
@@ -28,15 +27,12 @@
 from evalio.utils import pascal_to_snake
 
 Param = bool | int | float | str
-
-
-class ExperimentStatus(Enum):
-    Complete = "complete"
-    Fail = "fail"
-    Started = "started"
+"""A parameter value for a pipeline, can be a bool, int, float, or str."""
 
 
 class FailedMetadataParse(Exception):
+    """Exception raised when metadata parsing fails."""
+
     def __init__(self, reason: str):
         super().__init__(f"Failed to parse metadata: {reason}")
         self.reason = reason
@@ -44,6 +40,8 @@ def __init__(self, reason: str):
 
 @dataclass(kw_only=True)
 class Metadata:
+    """Base class for metadata associated with a trajectory."""
+
     file: Optional[Path] = None
     """File where the metadata was loaded to and from, if any."""
     _registry: ClassVar[dict[str, type[Self]]] = {}
@@ -53,26 +51,59 @@ def __init_subclass__(cls) -> None:
 
     @classmethod
     def tag(cls) -> str:
+        """Get the tag for the metadata class. Will be used for serialization and deserialization.
+
+        Returns:
+            The tag for the metadata class.
+        """
         return pascal_to_snake(cls.__name__)
 
     @classmethod
     def from_dict(cls, data: dict[str, Any]) -> Self:
+        """Create an instance of the metadata class from a dictionary.
+
+        Args:
+            data (dict[str, Any]): The dictionary containing the metadata.
+
+        Returns:
+            An instance of the metadata class.
+        """
         if "type" in data:
             del data["type"]
         return cls(**data)
 
     def to_dict(self) -> dict[str, Any]:
+        """Convert the metadata instance to a dictionary.
+
+        Returns:
+            The dictionary representation of the metadata.
+        """
         d = asdict(self)
         d["type"] = self.tag()  # add type tag for deserialization
         del d["file"]  # don't serialize the file path
         return d
 
     def to_yaml(self) -> str:
+        """Convert the metadata instance to a YAML string.
+
+        Returns:
+            The YAML representation of the metadata.
+        """
         data = self.to_dict()
         return yaml.safe_dump(data)
 
     @classmethod
     def from_yaml(cls, yaml_str: str) -> Metadata | FailedMetadataParse:
+        """Create an instance of the metadata class from a YAML string.
+
+        Will return the appropriate subclass based on the "type" field in the YAML.
+
+        Args:
+            yaml_str (str): The YAML string containing the metadata.
+
+        Returns:
+            An instance of the metadata class or an error.
+        """
         data = yaml.safe_load(yaml_str)
 
         if "type" not in data:
@@ -90,6 +121,8 @@ def from_yaml(cls, yaml_str: str) -> Metadata | FailedMetadataParse:
 
 @dataclass(kw_only=True)
 class GroundTruth(Metadata):
+    """Metadata for ground truth trajectories."""
+
     sequence: str
     """Dataset used to run the experiment."""
 
@@ -99,6 +132,8 @@ class GroundTruth(Metadata):
 
 @dataclass(kw_only=True)
 class Trajectory(Generic[M]):
+    """A trajectory of poses with associated timestamps and metadata."""
+
     stamps: list[Stamp] = field(default_factory=list)
     """List of timestamps for each pose."""
     poses: list[SE3] = field(default_factory=list)
@@ -113,15 +148,41 @@ def __post_init__(self):
             raise ValueError("Stamps and poses must have the same length.")
 
     def __getitem__(self, idx: int) -> tuple[Stamp, SE3]:
+        """Get a (stamp, pose) pair by index.
+
+        Args:
+            idx (int): The index of the (stamp, pose) pair.
+
+        Returns:
+            The (stamp, pose) pair at the given index.
+        """
         return self.stamps[idx], self.poses[idx]
 
     def __len__(self) -> int:
+        """Get the length of the trajectory.
+
+        Returns:
+            The number of (stamp, pose) pairs in the trajectory.
+        """
         return len(self.stamps)
 
-    def __iter__(self):
+    def __iter__(self) -> Iterator[tuple[Stamp, SE3]]:
+        """Iterate over the trajectory.
+
+        Returns:
+            An iterator over the (stamp, pose) pairs.
+        """
         return iter(zip(self.stamps, self.poses))
 
     def append(self, stamp: Stamp, pose: SE3):
+        """Append a new pose to the trajectory.
+
+        Will also write to file if the trajectory was opened with [open][evalio.types.Trajectory.open].
+
+        Args:
+            stamp (Stamp): The timestamp of the pose.
+            pose (SE3): The pose to append.
+        """
         self.stamps.append(stamp)
         self.poses.append(pose)
 
@@ -129,6 +190,11 @@ def append(self, stamp: Stamp, pose: SE3):
             self._csv_writer.writerow(self._serialize_pose(stamp, pose))
 
     def transform_in_place(self, T: SE3):
+        """Apply a transformation to all poses in the trajectory.
+
+        Args:
+            T (SE3): The transformation to apply.
+        """
         for i in range(len(self.poses)):
             self.poses[i] = self.poses[i] * T
 
@@ -158,7 +224,7 @@ def from_csv(
             skip_lines (int, optional): Number of lines to skip, useful for skipping headers. Defaults to 0.
 
         Returns:
-            Trajectory: Stored dataset
+            Stored trajectory
         """
         poses: list[SE3] = []
         stamps: list[Stamp] = []
@@ -197,14 +263,14 @@ def from_csv(
         return Trajectory(stamps=stamps, poses=poses)
 
     @staticmethod
-    def from_tum(path: Path) -> "Trajectory":
+    def from_tum(path: Path) -> Trajectory:
         """Load a TUM dataset pose file. Simple wrapper around [from_csv][evalio.types.Trajectory].
 
         Args:
             path (Path): Location of file.
 
         Returns:
-            Trajectory: Stored trajectory
+            Stored trajectory
         """
         return Trajectory.from_csv(path, ["sec", "x", "y", "z", "qx", "qy", "qz", "qw"])
 
@@ -220,7 +286,7 @@ def from_file(
             path (Path): Location of trajectory results.
 
         Returns:
-            Trajectory: Loaded trajectory with metadata, stamps, and poses.
+            Loaded trajectory with metadata, stamps, and poses.
         """
         if not path.exists():
             return FileNotFoundError(f"File {path} does not exist.")
@@ -249,12 +315,6 @@ def from_file(
 
     # ------------------------- Saving to file ------------------------- #
     def _serialize_pose(self, stamp: Stamp, pose: SE3) -> list[str | float]:
-        """Helper to serialize a stamped pose for csv writing.
-
-        Args:
-            stamp (Stamp): Timestamp associated with the pose.
-            pose (SE3): Pose to save.
-        """
         return [
             f"{stamp.sec}.{stamp.nsec:09}",
             pose.trans[0],
@@ -276,7 +336,6 @@ def _serialize_metadata(self) -> str:
 
     def _write(self):
         if self._file is None or self._csv_writer is None:
-            print_warning("Trajectory.write_experiment: No file is open.")
             return
 
         # write everything we've got so far
@@ -310,7 +369,7 @@ def open(self, path: Optional[Path] = None):
         self._write()
 
     def close(self):
-        """Close the CSV file if it is open with [write_experiment][evalio.types.Trajectory.write_experiment] and incremental writing."""
+        """Close the CSV file if it was opened with [open][evalio.types.Trajectory.open]."""
         if self._file is not None:
             self._file.close()
             self._file = None
@@ -319,6 +378,11 @@ def close(self):
             print_warning("Trajectory.close: No file to close.")
 
     def to_file(self, path: Optional[Path] = None):
+        """Save the trajectory to a CSV file.
+
+        Args:
+            path (Optional[Path], optional): Path to the CSV file. If not specified, utilizes the path in the metadata, if it exists. Defaults to None.
+        """
         self.open(path)
         self.close()
 
diff --git a/python/evalio/types/extended.py b/python/evalio/types/extended.py
index 61cb4976..c88cdd4b 100644
--- a/python/evalio/types/extended.py
+++ b/python/evalio/types/extended.py
@@ -14,6 +14,8 @@
 
 
 class ExperimentStatus(Enum):
+    """Status of the experiment."""
+
     Complete = "complete"
     Fail = "fail"
     Started = "started"
@@ -22,12 +24,18 @@ class ExperimentStatus(Enum):
 
 @dataclass(kw_only=True)
 class Experiment(Metadata):
+    """An experiment is a single run of a pipeline on a dataset.
+
+    It contains all the information needed to reproduce the run, including
+    the pipeline parameters, dataset, and status.
+    """
+
     name: str
     """Name of the experiment."""
     sequence: str | ds.Dataset
     """Dataset used to run the experiment."""
     sequence_length: int
-    """Length of the sequence, if set"""
+    """Length of the sequence"""
     pipeline: str | type[pl.Pipeline]
     """Pipeline used to generate the trajectory."""
     pipeline_version: str
@@ -62,9 +70,15 @@ def from_dict(cls, data: dict[str, Any]) -> Self:
 
     def setup(
         self,
-    ) -> (
-        tuple[pl.Pipeline, ds.Dataset] | ds.DatasetConfigError | pl.PipelineConfigError
-    ):
+    ) -> tuple[pl.Pipeline, ds.Dataset] | ds.SequenceNotFound | pl.PipelineNotFound:
+        """Setup the experiment by initializing the pipeline and dataset.
+
+        Args:
+            self (Experiment): The experiment instance.
+
+        Returns:
+            Tuple containing the initialized pipeline and dataset, or an error if the pipeline or dataset could not be found or configured.
+        """
         if isinstance(self.pipeline, str):
             ThisPipeline = pl.get_pipeline(self.pipeline)
             if isinstance(ThisPipeline, pl.PipelineNotFound):
diff --git a/python/evalio/utils.py b/python/evalio/utils.py
index bd1621d6..9fd72c2c 100644
--- a/python/evalio/utils.py
+++ b/python/evalio/utils.py
@@ -41,7 +41,7 @@ def pascal_to_snake(identifier: str) -> str:
         identifier (str): The PascalCase identifier to convert.
 
     Returns:
-        str: The converted snake_case identifier.
+        The converted snake_case identifier.
     """
     # Only split when going from lower to something else
     # this handles digits better than other approaches

From 312e8fa4c436fb6cf8d6dddb02bd3bcad6171ba6 Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Wed, 1 Oct 2025 15:07:18 -0400
Subject: [PATCH 18/40] Fix pipeline docs

---
 docs/ref/pipelines.md               |  9 +--------
 python/evalio/pipelines/__init__.py | 17 ++++++++++++++++-
 2 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/docs/ref/pipelines.md b/docs/ref/pipelines.md
index 71b7d482..bce64fda 100644
--- a/docs/ref/pipelines.md
+++ b/docs/ref/pipelines.md
@@ -2,11 +2,4 @@ For more information about the pipelines included in evalio, see the [included p
 
 ::: evalio.pipelines
     options:
-        members:
-            - Pipeline
-
-::: evalio.pipelines
-    options:
-        show_root_toc_entry: false
-        filters:
-            - "!Pipeline"
\ No newline at end of file
+        members_order: source
\ No newline at end of file
diff --git a/python/evalio/pipelines/__init__.py b/python/evalio/pipelines/__init__.py
index 6c912de3..233419ad 100644
--- a/python/evalio/pipelines/__init__.py
+++ b/python/evalio/pipelines/__init__.py
@@ -1,4 +1,12 @@
-from evalio._cpp.pipelines import *  # type: ignore  # noqa: F403
+from evalio._cpp.pipelines import (  # type: ignore
+    Pipeline,
+    CTICP,
+    KissICP,
+    GenZICP,
+    LOAM,
+    LioSAM,
+    MadICP,
+)
 
 from .parser import (
     register_pipeline,
@@ -15,6 +23,13 @@
 
 
 __all__ = [
+    "Pipeline",
+    "CTICP",
+    "KissICP",
+    "GenZICP",
+    "LOAM",
+    "LioSAM",
+    "MadICP",
     "all_pipelines",
     "get_pipeline",
     "register_pipeline",

From 760ed7cd6a8ab564cc7ac60ccfcd1dcb0e83bcf5 Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Wed, 1 Oct 2025 15:26:08 -0400
Subject: [PATCH 19/40] Clean up rest of pipeline docs

---
 docs/ref/pipelines.md               | 16 +++++++++++++++-
 python/evalio/pipelines/__init__.py | 25 +++++--------------------
 python/evalio/pipelines/parser.py   |  6 ++++++
 3 files changed, 26 insertions(+), 21 deletions(-)

diff --git a/docs/ref/pipelines.md b/docs/ref/pipelines.md
index bce64fda..42900e32 100644
--- a/docs/ref/pipelines.md
+++ b/docs/ref/pipelines.md
@@ -2,4 +2,18 @@ For more information about the pipelines included in evalio, see the [included p
 
 ::: evalio.pipelines
     options:
-        members_order: source
\ No newline at end of file
+        members:
+            - Pipeline
+            - CTICP
+            - KissICP
+            - GenZICP
+            - LOAM
+            - LioSAM
+            - MadICP
+            - PipelineNotFound
+            - UnusedPipelineParam
+            - InvalidPipelineParamType
+            - all_pipelines
+            - get_pipeline
+            - register_pipeline
+            - validate_params
\ No newline at end of file
diff --git a/python/evalio/pipelines/__init__.py b/python/evalio/pipelines/__init__.py
index 233419ad..27a36d66 100644
--- a/python/evalio/pipelines/__init__.py
+++ b/python/evalio/pipelines/__init__.py
@@ -1,35 +1,20 @@
-from evalio._cpp.pipelines import (  # type: ignore
-    Pipeline,
-    CTICP,
-    KissICP,
-    GenZICP,
-    LOAM,
-    LioSAM,
-    MadICP,
-)
+from evalio._cpp.pipelines import *  # type: ignore # noqa: F403
 
 from .parser import (
     register_pipeline,
     get_pipeline,
     all_pipelines,
     parse_config,
+    validate_params,
     PipelineNotFound,
-    InvalidPipelineConfig,
-    PipelineConfigError,
     UnusedPipelineParam,
     InvalidPipelineParamType,
-    validate_params,
+    InvalidPipelineConfig,
+    PipelineConfigError,
 )
 
-
 __all__ = [
-    "Pipeline",
-    "CTICP",
-    "KissICP",
-    "GenZICP",
-    "LOAM",
-    "LioSAM",
-    "MadICP",
+    "Pipeline",  # noqa: F405
     "all_pipelines",
     "get_pipeline",
     "register_pipeline",
diff --git a/python/evalio/pipelines/parser.py b/python/evalio/pipelines/parser.py
index 93d96bdc..de91e83e 100644
--- a/python/evalio/pipelines/parser.py
+++ b/python/evalio/pipelines/parser.py
@@ -17,6 +17,8 @@
 
 
 class PipelineNotFound(CustomException):
+    """Raised when a pipeline is not found in the registry."""
+
     def __init__(self, name: str):
         super().__init__(f"Pipeline '{name}' not found")
         self.name = name
@@ -29,6 +31,8 @@ def __init__(self, config: str):
 
 
 class UnusedPipelineParam(CustomException):
+    """Raised when a parameter is not used in the pipeline."""
+
     def __init__(self, param: str, pipeline: str):
         super().__init__(f"Parameter '{param}' is not used in pipeline '{pipeline}'")
         self.param = param
@@ -36,6 +40,8 @@ def __init__(self, param: str, pipeline: str):
 
 
 class InvalidPipelineParamType(CustomException):
+    """Raised when a parameter has an invalid type."""
+
     def __init__(self, param: str, expected_type: type, actual_type: type):
         super().__init__(
             f"Parameter '{param}' has invalid type. Expected '{expected_type.__name__}', got '{actual_type.__name__}'"

From 7f82c94bdd00f762a67c50de0a8b5974a1b588d1 Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Thu, 2 Oct 2025 09:10:49 -0400
Subject: [PATCH 20/40] Clean up some minor bugs

---
 python/evalio/cli/__init__.py |  3 +++
 python/evalio/cli/run.py      |  4 ++--
 python/evalio/cli/stats.py    | 22 +++++++++++++++++-----
 3 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/python/evalio/cli/__init__.py b/python/evalio/cli/__init__.py
index 22a16e49..1189397a 100644
--- a/python/evalio/cli/__init__.py
+++ b/python/evalio/cli/__init__.py
@@ -101,3 +101,6 @@ def global_options(
 __all__ = [
     "app",
 ]
+
+if __name__ == "__main__":
+    app()
diff --git a/python/evalio/cli/run.py b/python/evalio/cli/run.py
index 6cff857e..530e290b 100644
--- a/python/evalio/cli/run.py
+++ b/python/evalio/cli/run.py
@@ -212,7 +212,7 @@ def run_from_cli(
             pipeline=pipeline,
             pipeline_version=pipeline.version(),
             pipeline_params=params,
-            file=out / sequence / f"{name}.csv",
+            file=out / sequence.full_name / f"{name}.csv",
         )
         for sequence, length in datasets
         for name, pipeline, params in pipelines
@@ -271,7 +271,7 @@ def run(
             status = ty.ExperimentStatus.NotRun
 
         # Do something based on the status
-        info = f"{exp.pipeline.name()} on {exp.sequence}"
+        info = f"{exp.name} on {exp.sequence}"
         match status:
             case ty.ExperimentStatus.Complete:
                 print(f"Skipping {info}, already finished")
diff --git a/python/evalio/cli/stats.py b/python/evalio/cli/stats.py
index 032842a4..88f158e6 100644
--- a/python/evalio/cli/stats.py
+++ b/python/evalio/cli/stats.py
@@ -199,14 +199,14 @@ def evaluate_typer(
         ),
     ] = False,
     hide_columns: Annotated[
-        list[str],
+        Optional[list[str]],
         typer.Option(
             "-s",
-            "--show-columns",
-            help="Comma-separated list of columns to show.",
+            "--hide-columns",
+            help="Comma-separated list of columns to hide.",
             rich_help_panel="Output options",
         ),
-    ] = ["pipeline_version", "total_elapsed", "pipeline"],
+    ] = None,
     print_columns: Annotated[
         bool,
         typer.Option(
@@ -326,8 +326,20 @@ def evaluate_typer(
             c.print(f" - {col}")
         return
 
+    # hide some columns by default
+    if hide_columns is None:
+        hide_columns = []
+    hide_columns.extend(["pipeline_version", "total_elapsed", "pipeline"])
+
     # delete unneeded columns
-    remove_columns = [col for col in df.columns if df[col].drop_nulls().n_unique() == 1]
+    remove_columns = [
+        col
+        for col in df.columns
+        if col not in ["sequence", "name"]  # must keep these for later
+        and not col.startswith("RTE")  # want to keep metrics as well
+        and not col.startswith("ATE")
+        and df[col].drop_nulls().n_unique() == 1  # remove if they're all the same
+    ]
     remove_columns.extend([col for col in hide_columns if col in df.columns])
     df = df.drop(remove_columns)
 

From 12b2420afc3a6c124c0d0ca140614bfc66f708d3 Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Thu, 2 Oct 2025 14:29:51 -0400
Subject: [PATCH 21/40] Some stats optimizations

---
 cpp/bindings/types.h       |  9 +++++++++
 cpp/evalio/types.h         |  7 +++++++
 python/evalio/cli/stats.py | 20 +++++++++++++-------
 python/evalio/stats.py     | 35 +++++++++++++++++++----------------
 4 files changed, 48 insertions(+), 23 deletions(-)

diff --git a/cpp/bindings/types.h b/cpp/bindings/types.h
index 285777cf..4fadc7f2 100644
--- a/cpp/bindings/types.h
+++ b/cpp/bindings/types.h
@@ -2,6 +2,7 @@
 #include <nanobind/eigen/dense.h>
 #include <nanobind/nanobind.h>
 #include <nanobind/operators.h>
+#include <nanobind/stl/pair.h>
 #include <nanobind/stl/string.h>
 #include <nanobind/stl/tuple.h>
 #include <nanobind/stl/vector.h>
@@ -545,6 +546,14 @@ inline void makeTypes(nb::module_& m) {
     .def_ro("trans", &SE3::trans, "Translation as a 3D vector.")
     .def("toMat", &SE3::toMat, "Convert to a 4x4 matrix.")
     .def("inverse", &SE3::inverse, "Compute the inverse.")
+    .def_static(
+      "error",
+      &SE3::error,
+      "a"_a,
+      "b"_a,
+      "Compute the rotational (degrees) and translational (meters) error "
+      "between two SE3s as a tuple (rot, trans)."
+    )
     .def_static("exp", &SE3::exp, "xi"_a, "Create a SE3 from a 3D vector.")
     .def("log", &SE3::log, "Compute the logarithm of the transformation.")
     .def(nb::self * nb::self, "Compose two rigid body transformations.")
diff --git a/cpp/evalio/types.h b/cpp/evalio/types.h
index a8ddbe9a..e88a02db 100644
--- a/cpp/evalio/types.h
+++ b/cpp/evalio/types.h
@@ -388,6 +388,13 @@ struct SE3 {
     return SE3(inv_rot, inv_rot.rotate(-trans));
   }
 
+  static std::pair<double, double> error(const SE3& a, const SE3& b) {
+    auto delta = a.inverse() * b;
+    double rot_err = delta.rot.log().norm() * (180.0 / M_PI);
+    double trans_err = (delta.trans).norm();
+    return {rot_err, trans_err};
+  }
+
   static SE3 exp(const Eigen::Matrix<double, 6, 1>& xi) {
     Eigen::Vector3d omega = xi.head<3>();
     Eigen::Vector3d xyz = xi.tail<3>();
diff --git a/python/evalio/cli/stats.py b/python/evalio/cli/stats.py
index 88f158e6..14eefefa 100644
--- a/python/evalio/cli/stats.py
+++ b/python/evalio/cli/stats.py
@@ -1,6 +1,8 @@
+from copy import copy
 from pathlib import Path
 from typing import Annotated, Any, Callable, Optional, cast
 
+from evalio.types.base import Trajectory
 import polars as pl
 import itertools
 
@@ -86,17 +88,21 @@ def eval_dataset(
         del r["pipeline_params"]
 
         # add metrics
-        traj_aligned, gt_aligned = stats.align(traj, gt_og)
-        if length is not None and len(traj_aligned) > length:
-            traj_aligned.stamps = traj_aligned.stamps[:length]
-            traj_aligned.poses = traj_aligned.poses[:length]
+        gt_aligned = Trajectory(
+            stamps=[copy(s) for s in gt_og.stamps],
+            poses=[copy(p) for p in gt_og.poses],
+        )
+        stats.align(traj, gt_aligned, in_place=True)
+        if length is not None and len(traj) > length:
+            traj.stamps = traj.stamps[:length]
+            traj.poses = traj.poses[:length]
             gt_aligned.stamps = gt_aligned.stamps[:length]
             gt_aligned.poses = gt_aligned.poses[:length]
-        ate = stats.ate(traj_aligned, gt_aligned).summarize(metric)
+        ate = stats.ate(traj, gt_aligned).summarize(metric)
         r.update({"ATEt": ate.trans, "ATEr": ate.rot})
 
         for w in windows:
-            rte = stats.rte(traj_aligned, gt_aligned, w).summarize(metric)
+            rte = stats.rte(traj, gt_aligned, w).summarize(metric)
             r.update({f"RTEt_{w.name()}": rte.trans, f"RTEr_{w.name()}": rte.rot})
 
         results.append(r)
@@ -104,7 +110,7 @@ def eval_dataset(
         if rr is not None and convert is not None and colors is not None and visualize:
             rr.log(
                 traj.metadata.name,
-                convert(traj_aligned, color=colors[index]),
+                convert(traj, color=colors[index]),
                 static=True,
             )
 
diff --git a/python/evalio/stats.py b/python/evalio/stats.py
index 1b7a4ba7..d3da2b4d 100644
--- a/python/evalio/stats.py
+++ b/python/evalio/stats.py
@@ -202,12 +202,15 @@ def align_stamps(traj1: ty.Trajectory[M1], traj2: ty.Trajectory[M2]):
         traj1, traj2 = traj2, traj1  # type: ignore
         swapped = True
 
+    # cache this value
+    len_traj1 = len(traj1)
+
     # Align the two trajectories by subsampling keeping traj1 stamps
     traj1_idx = 0
     traj1_stamps: list[ty.Stamp] = []
     traj1_poses: list[ty.SE3] = []
     for i, stamp in enumerate(traj2.stamps):
-        while traj1_idx < len(traj1) - 1 and traj1.stamps[traj1_idx] < stamp:
+        while traj1_idx < len_traj1 - 1 and traj1.stamps[traj1_idx] < stamp:
             traj1_idx += 1
 
         # go back one if we overshot
@@ -217,7 +220,7 @@ def align_stamps(traj1: ty.Trajectory[M1], traj2: ty.Trajectory[M2]):
         traj1_stamps.append(traj1.stamps[traj1_idx])
         traj1_poses.append(traj1.poses[traj1_idx])
 
-        if traj1_idx >= len(traj1) - 1:
+        if traj1_idx >= len_traj1 - 1:
             traj2.stamps = traj2.stamps[: i + 1]
             traj2.poses = traj2.poses[: i + 1]
             break
@@ -244,10 +247,7 @@ def _compute_metric(gts: list[ty.SE3], poses: list[ty.SE3]) -> Error:
     error_t = np.zeros(len(gts))
     error_r = np.zeros(len(gts))
     for i, (gt, pose) in enumerate(zip(gts, poses)):
-        delta = gt.inverse() * pose
-        error_t[i] = np.sqrt(delta.trans @ delta.trans)
-        r_diff = delta.rot.log()
-        error_r[i] = np.sqrt(r_diff @ r_diff) * 180 / np.pi
+        error_r[i], error_t[i] = ty.SE3.error(gt, pose)
 
     return Error(rot=error_r, trans=error_t)
 
@@ -322,16 +322,19 @@ def rte(
     window_deltas_poses: list[ty.SE3] = []
     window_deltas_gts: list[ty.SE3] = []
 
+    # cache this value
+    len_gt = len(gt)
+
     if isinstance(window, WindowSeconds):
         # Find our pairs for computation
         end_idx = 1
         duration = ty.Duration.from_sec(window.value)
 
-        for i in range(len(gt)):
-            while end_idx < len(gt) and gt.stamps[end_idx] - gt.stamps[i] < duration:
+        for i in range(len_gt):
+            while end_idx < len_gt and gt.stamps[end_idx] - gt.stamps[i] < duration:
                 end_idx += 1
 
-            if end_idx >= len(gt):
+            if end_idx >= len_gt:
                 break
 
             window_deltas_poses.append(traj.poses[i].inverse() * traj.poses[end_idx])
@@ -341,8 +344,8 @@ def rte(
 
     elif isinstance(window, WindowMeters):
         # Compute deltas for all of ground truth poses
-        dist = np.zeros(len(gt))
-        for i in range(1, len(gt)):
+        dist = np.zeros(len_gt)
+        for i in range(1, len_gt):
             diff: NDArray[np.float64] = gt.poses[i].trans - gt.poses[i - 1].trans
             dist[i] = np.sqrt(diff @ diff)
 
@@ -351,11 +354,11 @@ def rte(
         end_idx_prev = 0
 
         # Find our pairs for computation
-        for i in range(len(gt)):
-            while end_idx < len(gt) and cum_dist[end_idx] - cum_dist[i] < window.value:
+        for i in range(len_gt):
+            while end_idx < len_gt and cum_dist[end_idx] - cum_dist[i] < window.value:
                 end_idx += 1
 
-            if end_idx >= len(gt):
+            if end_idx >= len_gt:
                 break
             elif end_idx == end_idx_prev:
                 continue
@@ -368,10 +371,10 @@ def rte(
     if len(window_deltas_poses) == 0:
         if isinstance(traj.metadata, ty.Experiment):
             print_warning(
-                f"No windows found with size {window} for '{traj.metadata.name}' on '{traj.metadata.sequence}'"
+                f"No {window} windows found for '{traj.metadata.name}' on '{traj.metadata.sequence}'"
             )
         else:
-            print_warning(f"No windows found with size {window}")
+            print_warning(f"No {window} windows found")
         return Error(rot=np.array([np.nan]), trans=np.array([np.nan]))
 
     # Compute the RTE

From 12424a7b0a7a4fc3d0b600a9d50ad62c195d6aca Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Thu, 2 Oct 2025 14:38:08 -0400
Subject: [PATCH 22/40] preprocess stamp style in csv loading

---
 python/evalio/types/base.py | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/python/evalio/types/base.py b/python/evalio/types/base.py
index c7cdc0fa..c7f02f4e 100644
--- a/python/evalio/types/base.py
+++ b/python/evalio/types/base.py
@@ -229,6 +229,11 @@ def from_csv(
         poses: list[SE3] = []
         stamps: list[Stamp] = []
 
+        # Pre-check how stamps are stored for efficiency
+        swap_sec_t = "t" in fieldnames
+        has_sec_field = "sec" in fieldnames or "t" in fieldnames
+        has_nsec_field = "nsec" in fieldnames
+
         with open(path) as f:
             csvfile = list(filter(lambda row: row[0] != "#", f))
             if skip_lines is not None:
@@ -244,19 +249,22 @@ def from_csv(
                 t = np.array([float(line["x"]), float(line["y"]), float(line["z"])])
                 pose = SE3(r, t)
 
-                if "t" in fieldnames:
+                if swap_sec_t:
                     line["sec"] = line["t"]
 
-                if "nsec" not in fieldnames:
-                    s, ns = line["sec"].split(
-                        "."
-                    )  # parse separately to get exact stamp
-                    ns = ns.ljust(9, "0")  # pad to 9 digits for nanoseconds
-                    stamp = Stamp(sec=int(s), nsec=int(ns))
-                elif "sec" not in fieldnames:
-                    stamp = Stamp.from_nsec(int(line["nsec"]))
-                else:
-                    stamp = Stamp(sec=int(line["sec"]), nsec=int(line["nsec"]))
+                match (has_sec_field, has_nsec_field):
+                    case (True, True):
+                        stamp = Stamp(sec=int(line["sec"]), nsec=int(line["nsec"]))
+                    case (True, False):
+                        # parse separately to get exact stamp
+                        s, ns = line["sec"].split(".")
+                        ns = ns.ljust(9, "0")  # pad to 9 digits for nanoseconds
+                        stamp = Stamp(sec=int(s), nsec=int(ns))
+                    case (False, True):
+                        stamp = Stamp.from_nsec(int(line["nsec"]))
+                    case (False, False):
+                        raise ValueError("Must have at least one of 'sec' or 'nsec'.")
+
                 poses.append(pose)
                 stamps.append(stamp)
 

From a8d8dea6cb92f75a71c33b53afaf14d1a6a3c86c Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Thu, 2 Oct 2025 17:42:57 -0400
Subject: [PATCH 23/40] Shorten readme, add in citations

---
 CITATION.bib |   9 +++++
 README.md    | 112 +++++++++++----------------------------------------
 2 files changed, 32 insertions(+), 89 deletions(-)
 create mode 100644 CITATION.bib

diff --git a/CITATION.bib b/CITATION.bib
new file mode 100644
index 00000000..361a4383
--- /dev/null
+++ b/CITATION.bib
@@ -0,0 +1,9 @@
+@misc{potokar2025_evaluation_lidar_odometry,
+  title         = {A Comprehensive Evaluation of LiDAR Odometry Techniques},
+  author        = {Easton Potokar and Michael Kaess},
+  year          = {2025},
+  eprint        = {2507.16000},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.RO},
+  url           = {https://arxiv.org/abs/2507.16000}
+}
\ No newline at end of file
diff --git a/README.md b/README.md
index f4f51d69..c38ea214 100644
--- a/README.md
+++ b/README.md
@@ -13,15 +13,15 @@ Specifically, it provides a common interface for connecting LIO datasets and LIO
 
 ## Installation
 
-evalio is available on PyPi, so simply install via your favorite python package manager,
+evalio is available on PyPi (with all pipelines compiled in!), so simply install via your favorite python package manager,
 ```bash
 uv add evalio      # uv
 pip install evalio # pip
 ```
 
-## Usage
+## Basic Usage
 
-evalio can be used both as a python library and as a CLI for both datasets and pipelines.
+evalio can be used both as a python library and as a CLI for both datasets and pipelines. We cover just the tip of the iceberg here, so please check out the [docs](https://contagon.github.io/evalio/) for more information.
 
 ### Datasets
 
@@ -30,67 +30,24 @@ Once evalio is installed, datasets can be listed and downloaded via the CLI inte
 evalio ls datasets
 evalio download hilti_2022/basement_2
 ```
-evalio downloads data to the `EVALIO_DATA` environment variable, or if unset to the local folder `./evalio_data`. All the trajectories in a dataset can also be downloaded by using the wildcard `hilti_2022/*`, making sure to escape the asterisk as needed.
-
-> [!TIP]
-> evalio also comes with autocomplete, which makes typing the long dataset and pipeline names much easier. To install, do one of the following,
-> ```bash
-> eval "$(evalio --show-completion)" # install for the current session
-> evalio --install-completion        # install for all future sessions
-
-> [!NOTE]
-> Many datasets use [gdown](https://github.com/wkentaro/gdown) to download datasets from google drive. Unfortunately, this can occasionally be finicky due to google's download limits, however [downloading cookies from your browser](https://github.com/wkentaro/gdown?tab=readme-ov-file#i-set-the-permission-anyone-with-link-but-i-still-cant-download) can often help.
-
 
 Once downloaded, a trajectory can then be easily used in python,
 ```python
-from evalio.datasets import Hilti2022
+from evalio import datasets as ds
 
 # for all data
-for mm in Hilti2022.basement_2:
+for mm in ds.Hilti2022.basement_2:
     print(mm)
 
 # for lidars
-for scan in Hilti2022.basement_2.lidar():
+for scan in ds.Hilti2022.basement_2.lidar():
     print(scan)
 
 # for imu
-for imu in Hilti2022.basement_2.imu():
+for imu in ds.Hilti2022.basement_2.imu():
     print(imu)
 ```
 
-For example, you can easily get a single scan to plot a bird-eye view,
-```python
-import matplotlib.pyplot as plt
-import numpy as np
-
-# get the 10th scan
-scan = Hilti2022.basement_2.get_one_lidar(10)
-# always in row-major order, with stamp at start of scan
-x = np.array([p.x for p in scan.points])
-y = np.array([p.y for p in scan.points])
-z = np.array([p.z for p in scan.points])
-plt.scatter(x, y, c=z, s=1)
-plt.axis('equal')
-plt.show()
-```
-evalio also comes with a built wrapper for converting to [rerun](rerun.io) types,
-```python
-import rerun as rr
-from evalio.rerun import convert
-
-rr.init("evalio")
-rr.connect_tcp()
-for scan in Hilti2022.basement_2.lidar():
-    rr.set_time("timeline", timestamp=scan.stamp.to_sec())
-    rr.log("lidar", convert(scan, color=[255, 0, 255]))
-```
-
-> [!NOTE]  
-> To run the rerun visualization, rerun must be installed. This can be done by installing `rerun-sdk` or `evalio[vis]` from PyPi.
-
-We recommend checking out the [base dataset class](python/evalio/datasets/base.py) for more information on how to interact with datasets.
-
 ### Pipelines
 
 The other half of evalio is the pipelines that can be run on various datasets. All pipelines and their parameters can be shown via,
@@ -105,15 +62,13 @@ This will run the pipeline on the dataset and save the results to the `results`
 ```bash
 evalio stats results
 ```
-> [!NOTE]  
-> KissICP does poorly by default on hilti_2022/basement_2, due to the close range and large default voxel size. You can visualize this by adding `-vvv` to the `run` command to visualize the trajectory in rerun.
 
 More complex experiments can be run, including varying pipeline parameters, via specifying a config file,
 ```yaml
 output_dir: ./results/
 
 datasets:
-  # Run on all of newer college trajectories
+  # Run on all of hilti trajectories
   - hilti_2022/*
   # Run on first 1000 scans of multi campus
   - name: multi_campus/ntu_day_01
@@ -126,7 +81,7 @@ pipelines:
   - name: kiss_tweaked
     pipeline: kiss
     deskew: true
-    # Some of these datasets need smaller voxel sizes
+    # Sweep over voxel size parameter
     sweep:
       voxel_size: [0.1, 0.5, 1.0]
       
@@ -135,43 +90,22 @@ This can then be run via
 ```bash
 evalio run -c config.yml
 ```
-That's about the gist of it! Try playing around the CLI interface to see what else is possible, such as a number of visualization options using rerun. Feel free to open an issue if you have any questions, suggestions, or problems.
-
-## Custom Datasets & Pipelines
-We understand that using an internal or work-in-progress datasets and pipelines will often be needed, thus evalio has full support for this. As mentioned above, we recommend checking out our [example](https://github.com/contagon/evalio-example) for more information how to to do this (it's pretty easy!). 
-
-The TL;DR version, a custom dataset can be made via inheriting from the `Dataset` class in python only, and a custom pipeline from inheriting the `Pipeline` class in either C++ or python. These can then be made available to evalio via the `EVALIO_CUSTOM` env variable point to the python module that contains them.
-
-We **highly** recommend making a PR to merge your custom datasets or pipelines into evalio once they are ready. This will make it more likely the community will use and cite your work, as well as increase the usefulness of evalio for everyone.
-
-## Building from Source
-
-While we recommend simply installing the python package using your preferred python package manager (our is `uv`), we've attempted to make building from source as easy as possible. We generally build through [scikit-core-build](https://scikit-build-core.readthedocs.io/) which provides a simple wrapper for building CMake projects as python packages. `uv` is our frontend of choice for this process, but it is also possible via pip
-```bash
-uv sync          # uv version
-pip install -e . # pip version
-```
-
-Of course, building via the usual `CMake` way is also possible, with the only default dependency being `Eigen3`,
-```bash
-mkdir build
-cd build
-cmake ..
-make
-```
-
-By default, all pipelines are not included due to their large dependencies. CMake will look for them in the `cpp/bindings/pipelines-src` directory. If you'd like to add them, simply run the `clone_pipelines.sh` script that will clone and patch them appropriately. 
-
-When these pipelines are included, the number of dependencies increases significantly, so have provided a [docker image](https://github.com/contagon/evalio/pkgs/container/evalio_manylinux_2_28_x86_64) that includes all dependencies for building as well as a VSCode devcontainer configuration. When opening in VSCode, you'll automatically be prompted to open in this container.
 
 ## Contributing
 
 Contributions are always welcome! Feel free to open an issue, pull request, etc. We're happy to help you get started. The following are rough instructions for specifically adding additional datasets or pipelines.
 
-### Datasets
-Datasets are easy to add, simply drop your file into the [python/evalio/datasets](python/evalio/datasets/) folder, and add it into the [init](python/evalio/datasets/__init__.py) file.
-
-### Pipelines
-If adding in a python pipeline, it's near identical to adding a dataset. Drop your file into the [python/evalio/pipelines](python/evalio/pipelines/) folder, and add it into the [init](python/evalio/pipelines/__init__.py) file.
-
-C++ pipelines are more involved (and probably worth the effort). Your header file belongs in the [cpp/bindings/pipelines](cpp/bindings/pipelines/) folder. To get it to build, make sure it's added to [clone_pipelines.sh](clone_pipelines.sh), the proper [CMakeLists.txt](cpp/bindings/CMakeLists.txt), and the [bindings.h] header. Finally, make sure all dependencies are also added to the docker build script, found in the [docker](docker/) folder.
\ No newline at end of file
+## Citation
+
+If you use evalio in your research, please cite the following paper,
+```bibtex
+@misc{potokar2025_evaluation_lidar_odometry,
+      title={A Comprehensive Evaluation of LiDAR Odometry Techniques}, 
+      author={Easton Potokar and Michael Kaess},
+      year={2025},
+      eprint={2507.16000},
+      archivePrefix={arXiv},
+      primaryClass={cs.RO},
+      url={https://arxiv.org/abs/2507.16000}, 
+}
+```
\ No newline at end of file

From 20bd7e30f6c99c6026ab29adbdfc270420249b0e Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Thu, 2 Oct 2025 17:46:47 -0400
Subject: [PATCH 24/40] Try with cff file

---
 CITATION.bib |  9 ---------
 CITATION.cff | 23 +++++++++++++++++++++++
 README.md    |  2 +-
 3 files changed, 24 insertions(+), 10 deletions(-)
 delete mode 100644 CITATION.bib
 create mode 100644 CITATION.cff

diff --git a/CITATION.bib b/CITATION.bib
deleted file mode 100644
index 361a4383..00000000
--- a/CITATION.bib
+++ /dev/null
@@ -1,9 +0,0 @@
-@misc{potokar2025_evaluation_lidar_odometry,
-  title         = {A Comprehensive Evaluation of LiDAR Odometry Techniques},
-  author        = {Easton Potokar and Michael Kaess},
-  year          = {2025},
-  eprint        = {2507.16000},
-  archiveprefix = {arXiv},
-  primaryclass  = {cs.RO},
-  url           = {https://arxiv.org/abs/2507.16000}
-}
\ No newline at end of file
diff --git a/CITATION.cff b/CITATION.cff
new file mode 100644
index 00000000..6140483a
--- /dev/null
+++ b/CITATION.cff
@@ -0,0 +1,23 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+- family-names: "Potokar"
+  given-names: "Easton"
+- family-names: "Kaess"
+  given-names: "Michael"
+title: "evalio"
+date-released: 2025
+url: "https://arxiv.org/abs/2507.16000"
+preferred-citation:
+  type: misc
+  authors:
+  - family-names: "Potokar"
+    given-names: "Easton"
+  - family-names: "Kaess"
+    given-names: "Michael"
+  title: "A Comprehensive Evaluation of LiDAR Odometry Techniques"
+  year: 2025
+  eprint: "2507.16000"
+  archivePrefix: "arXiv"
+  primaryClass: "cs.RO"
+  url: "https://arxiv.org/abs/2507.16000"
\ No newline at end of file
diff --git a/README.md b/README.md
index c38ea214..3468fbcd 100644
--- a/README.md
+++ b/README.md
@@ -93,7 +93,7 @@ evalio run -c config.yml
 
 ## Contributing
 
-Contributions are always welcome! Feel free to open an issue, pull request, etc. We're happy to help you get started. The following are rough instructions for specifically adding additional datasets or pipelines.
+Contributions are always welcome! Feel free to open an issue, pull request, etc. The documentation has a more details on developing new datasets and pipelines.
 
 ## Citation
 

From 400b93359b3c8747c32c7396ef6a124d3039d4b0 Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Thu, 2 Oct 2025 18:00:50 -0400
Subject: [PATCH 25/40] Fix citation.. hopefully

---
 CITATION.cff | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/CITATION.cff b/CITATION.cff
index 6140483a..bb9e091b 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -1,23 +1,27 @@
 cff-version: 1.2.0
-message: "If you use this software, please cite it as below."
+message: "If you use this software please cite it as below."
 authors:
 - family-names: "Potokar"
   given-names: "Easton"
 - family-names: "Kaess"
   given-names: "Michael"
 title: "evalio"
-date-released: 2025
+date-released: 2025-07-21
 url: "https://arxiv.org/abs/2507.16000"
 preferred-citation:
-  type: misc
+  type: report
   authors:
   - family-names: "Potokar"
     given-names: "Easton"
   - family-names: "Kaess"
     given-names: "Michael"
   title: "A Comprehensive Evaluation of LiDAR Odometry Techniques"
+  doi: "10.48550/arXiv.2507.16000"
+  institution: 
+    name: "arXiv preprint"
+  number: "arXiv:2507.16000"
+  publisher: 
+    name: "arXiv"
+  day: 21
+  month: 7
   year: 2025
-  eprint: "2507.16000"
-  archivePrefix: "arXiv"
-  primaryClass: "cs.RO"
-  url: "https://arxiv.org/abs/2507.16000"
\ No newline at end of file

From e875cfd3c190b0cbfb28a6a7cc4f251a894e130e Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Thu, 2 Oct 2025 19:52:13 -0400
Subject: [PATCH 26/40] Move to bib for citation

---
 CITATION.bib |  9 +++++++++
 CITATION.cff | 27 ---------------------------
 2 files changed, 9 insertions(+), 27 deletions(-)
 create mode 100644 CITATION.bib
 delete mode 100644 CITATION.cff

diff --git a/CITATION.bib b/CITATION.bib
new file mode 100644
index 00000000..361a4383
--- /dev/null
+++ b/CITATION.bib
@@ -0,0 +1,9 @@
+@misc{potokar2025_evaluation_lidar_odometry,
+  title         = {A Comprehensive Evaluation of LiDAR Odometry Techniques},
+  author        = {Easton Potokar and Michael Kaess},
+  year          = {2025},
+  eprint        = {2507.16000},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.RO},
+  url           = {https://arxiv.org/abs/2507.16000}
+}
\ No newline at end of file
diff --git a/CITATION.cff b/CITATION.cff
deleted file mode 100644
index bb9e091b..00000000
--- a/CITATION.cff
+++ /dev/null
@@ -1,27 +0,0 @@
-cff-version: 1.2.0
-message: "If you use this software please cite it as below."
-authors:
-- family-names: "Potokar"
-  given-names: "Easton"
-- family-names: "Kaess"
-  given-names: "Michael"
-title: "evalio"
-date-released: 2025-07-21
-url: "https://arxiv.org/abs/2507.16000"
-preferred-citation:
-  type: report
-  authors:
-  - family-names: "Potokar"
-    given-names: "Easton"
-  - family-names: "Kaess"
-    given-names: "Michael"
-  title: "A Comprehensive Evaluation of LiDAR Odometry Techniques"
-  doi: "10.48550/arXiv.2507.16000"
-  institution: 
-    name: "arXiv preprint"
-  number: "arXiv:2507.16000"
-  publisher: 
-    name: "arXiv"
-  day: 21
-  month: 7
-  year: 2025

From 642a7aafd38968d15cbd611861fdae05cf6e2aff Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Thu, 2 Oct 2025 20:18:00 -0400
Subject: [PATCH 27/40] Clean up stats options

---
 python/evalio/cli/run.py        |  2 +-
 python/evalio/cli/stats.py      | 81 +++++++++++++++++++++++++--------
 python/evalio/types/extended.py |  2 +-
 tests/test_io.py                |  2 +-
 4 files changed, 65 insertions(+), 22 deletions(-)

diff --git a/python/evalio/cli/run.py b/python/evalio/cli/run.py
index 530e290b..d55e0708 100644
--- a/python/evalio/cli/run.py
+++ b/python/evalio/cli/run.py
@@ -363,6 +363,6 @@ def run_single(
     loop.close()
     traj.metadata.status = ty.ExperimentStatus.Complete
     traj.metadata.total_elapsed = time_total
-    traj.metadata.max_step_elapsed = time_max
+    traj.metadata.max_elapsed = time_max
     traj.rewrite()
     traj.close()
diff --git a/python/evalio/cli/stats.py b/python/evalio/cli/stats.py
index 14eefefa..359f437a 100644
--- a/python/evalio/cli/stats.py
+++ b/python/evalio/cli/stats.py
@@ -82,10 +82,7 @@ def eval_dataset(
     # Iterate over each
     results: list[dict[str, Any]] = []
     for index, traj in enumerate(all_trajs):
-        r = traj.metadata.to_dict()
-        # flatten pipeline params
-        r.update(r["pipeline_params"])
-        del r["pipeline_params"]
+        r: dict[str, Any] = {}
 
         # add metrics
         gt_aligned = Trajectory(
@@ -98,13 +95,22 @@ def eval_dataset(
             traj.poses = traj.poses[:length]
             gt_aligned.stamps = gt_aligned.stamps[:length]
             gt_aligned.poses = gt_aligned.poses[:length]
-        ate = stats.ate(traj, gt_aligned).summarize(metric)
-        r.update({"ATEt": ate.trans, "ATEr": ate.rot})
 
         for w in windows:
             rte = stats.rte(traj, gt_aligned, w).summarize(metric)
             r.update({f"RTEt_{w.name()}": rte.trans, f"RTEr_{w.name()}": rte.rot})
 
+        ate = stats.ate(traj, gt_aligned).summarize(metric)
+        r.update({"ATEt": ate.trans, "ATEr": ate.rot})
+
+        # add metadata
+        r |= traj.metadata.to_dict()
+        # flatten pipeline params
+        r.update(r["pipeline_params"])
+        del r["pipeline_params"]
+        # remove type tag
+        del r["type"]
+
         results.append(r)
 
         if rr is not None and convert is not None and colors is not None and visualize:
@@ -205,11 +211,20 @@ def evaluate_typer(
         ),
     ] = False,
     hide_columns: Annotated[
+        Optional[list[str]],
+        typer.Option(
+            "-h",
+            "--hide",
+            help="Columns to hide, may be repeated.",
+            rich_help_panel="Output options",
+        ),
+    ] = None,
+    show_columns: Annotated[
         Optional[list[str]],
         typer.Option(
             "-s",
-            "--hide-columns",
-            help="Comma-separated list of columns to hide.",
+            "--show",
+            help="Columns to force show, may be repeated.",
             rich_help_panel="Output options",
         ),
     ] = None,
@@ -242,7 +257,6 @@ def evaluate_typer(
         stats.MetricKind,
         typer.Option(
             "--metric",
-            "-m",
             help="Metric to use for ATE/RTE computation. Defaults to sse.",
             rich_help_panel="Metric options",
         ),
@@ -321,9 +335,8 @@ def evaluate_typer(
 
     # clean up timing
     df = df.with_columns(
-        ((pl.col("sequence_length") / pl.col("total_elapsed")).alias("Hz"))
+        ((pl.col("sequence_length") / pl.col("total_elapsed")).alias("hz"))
     )
-    df = df.rename({"max_step_elapsed": "Max (s)"})
 
     # print columns if requested
     if print_columns:
@@ -332,23 +345,53 @@ def evaluate_typer(
             c.print(f" - {col}")
         return
 
-    # hide some columns by default
-    if hide_columns is None:
-        hide_columns = []
-    hide_columns.extend(["pipeline_version", "total_elapsed", "pipeline"])
+    # iterate through pipelines, finding unneeded columns
+    unused_columns: set[str] = set()
+    for pipeline in df["pipeline"].unique():
+        df_pipeline = df.filter(pl.col("pipeline") == pipeline)
+        unused_columns = unused_columns.union(
+            col
+            for col in df_pipeline.columns
+            if df_pipeline[col].drop_nulls().n_unique() == 1
+        )
+
+    # add in a few more that we usually shouldn't need
+    unused_columns.add("total_elapsed")
+    unused_columns.add("pipeline")
 
-    # delete unneeded columns
     remove_columns = [
         col
-        for col in df.columns
+        for col in unused_columns
         if col not in ["sequence", "name"]  # must keep these for later
         and not col.startswith("RTE")  # want to keep metrics as well
         and not col.startswith("ATE")
-        and df[col].drop_nulls().n_unique() == 1  # remove if they're all the same
     ]
-    remove_columns.extend([col for col in hide_columns if col in df.columns])
+
+    # forcibly hide / show some columns
+    if hide_columns is not None:
+        for col in hide_columns:
+            if col not in df.columns:
+                print_warning(f"Column {col} not found, cannot hide.")
+            else:
+                remove_columns.append(col)
+
+    if show_columns is not None:
+        for col in show_columns:
+            if col not in df.columns:
+                print_warning(f"Column {col} not found, cannot show.")
+            elif col in remove_columns:
+                remove_columns.remove(col)
+
     df = df.drop(remove_columns)
 
+    # rearrange for a more useful ordering (name to the left)
+    cols = df.columns
+    if "pipeline" in cols:
+        cols.insert(0, cols.pop(cols.index("pipeline")))
+    if "name" in cols:
+        cols.insert(0, cols.pop(cols.index("name")))
+    df = df.select(cols)
+
     # sort if requested
     if sort not in df.columns:
         print_warning(f"Column {sort} not found, cannot sort.")
diff --git a/python/evalio/types/extended.py b/python/evalio/types/extended.py
index c88cdd4b..ffb0f54b 100644
--- a/python/evalio/types/extended.py
+++ b/python/evalio/types/extended.py
@@ -46,7 +46,7 @@ class Experiment(Metadata):
     """Status of the experiment, e.g. "success", "failure", etc."""
     total_elapsed: Optional[float] = None
     """Total time taken for the experiment, as a string."""
-    max_step_elapsed: Optional[float] = None
+    max_elapsed: Optional[float] = None
     """Maximum time taken for a single step in the experiment, as a string."""
 
     def to_dict(self) -> dict[str, Any]:
diff --git a/tests/test_io.py b/tests/test_io.py
index aa00d74b..c6ee043f 100644
--- a/tests/test_io.py
+++ b/tests/test_io.py
@@ -13,7 +13,7 @@ def make_exp() -> ty.Experiment:
         pipeline_version="0.1.0",
         pipeline_params={"param1": 1, "param2": "value"},
         total_elapsed=10.5,
-        max_step_elapsed=0.24,
+        max_elapsed=0.24,
     )
 
 

From 7af2459ac562f1c19c99b8a4043fa9959b657505 Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Fri, 3 Oct 2025 10:20:00 -0400
Subject: [PATCH 28/40] Add in faster csv parser

---
 cpp/bindings/ros_pc2.h      | 76 +++++++++++++++++++++++++++++++++++++
 python/evalio/types/base.py | 44 ++++-----------------
 tests/test_io.py            | 55 +++++++++++++++++++++++++++
 3 files changed, 139 insertions(+), 36 deletions(-)

diff --git a/cpp/bindings/ros_pc2.h b/cpp/bindings/ros_pc2.h
index a6743939..18839f15 100644
--- a/cpp/bindings/ros_pc2.h
+++ b/cpp/bindings/ros_pc2.h
@@ -8,6 +8,7 @@
 #include <cmath>
 #include <cstddef>
 #include <fstream>
+#include <map>
 
 #include "evalio/types.h"
 
@@ -386,6 +387,79 @@ inline LidarMeasurement helipr_bin_to_evalio(
   return mm;
 }
 
+/// Parse a CSV line into an SE3 object. The idx map should contain the indices
+/// of the required fields: "qw", "qx", "qy", "qz", "x", "y", "z".
+inline std::pair<Stamp, SE3> parse_csv_line(
+  const std::string& s,
+  const char delimiter,
+  const std::map<std::string, int>& idx
+) {
+  std::stringstream ss(s);
+  std::string item;
+  std::vector<std::string> elems;
+  while (std::getline(ss, item, delimiter)) {
+    elems.push_back(item);
+  }
+
+  // Parse out the fields
+  SO3 r = SO3 {
+    .qx = std::stod(elems[idx.at("qx")]),
+    .qy = std::stod(elems[idx.at("qy")]),
+    .qz = std::stod(elems[idx.at("qz")]),
+    .qw = std::stod(elems[idx.at("qw")]),
+  };
+  Eigen::Vector3d t = Eigen::Vector3d(
+    std::stod(elems[idx.at("x")]),
+    std::stod(elems[idx.at("y")]),
+    std::stod(elems[idx.at("z")])
+  );
+
+  Stamp stamp;
+  // If both sec/nsec are given
+  if (idx.count("sec") && idx.count("nsec")) {
+    stamp = Stamp {
+      .sec = static_cast<uint32_t>(std::stoul(elems[idx.at("sec")])),
+      .nsec = static_cast<uint32_t>(std::stoul(elems[idx.at("nsec")]))
+    };
+  }
+
+  // If only sec is given, split it into sec/nsec
+  else if (idx.count("sec")) {
+    // Find decimal place
+    std::string sec_str = elems[idx.at("sec")];
+    size_t dot_pos = sec_str.find('.');
+    if (dot_pos == std::string::npos) {
+      throw std::runtime_error("Failed to find decimal in sec field.");
+    }
+
+    // extract sec
+    uint32_t sec_part = std::stoul(sec_str.substr(0, dot_pos));
+
+    // extract & pad nsec
+    std::string nsec_str = sec_str.substr(dot_pos + 1);
+    if (nsec_str.size() > 9) {
+      throw std::runtime_error("Too many digits in fractional part of sec.");
+    } else if (nsec_str.size() < 9) {
+      nsec_str += std::string(9 - nsec_str.size(), '0');
+    }
+    uint32_t nsec_part = std::stoul(nsec_str);
+
+    stamp = Stamp {.sec = sec_part, .nsec = nsec_part};
+  }
+
+  // If only nsec is given
+  else if (idx.count("nsec")) {
+    stamp = Stamp::from_nsec(std::stoul(elems[idx.at("nsec")]));
+  }
+
+  // If neither is given, throw an error
+  else {
+    throw std::runtime_error("Must have at least one of 'sec' or 'nsec'.");
+  }
+
+  return std::make_pair(stamp, SE3(r, t));
+}
+
 // ---------------------- Create python bindings ---------------------- //
 inline void makeConversions(nb::module_& m) {
   nb::enum_<DataType>(m, "DataType")
@@ -454,6 +528,8 @@ inline void makeConversions(nb::module_& m) {
   m.def("helipr_bin_to_evalio", &helipr_bin_to_evalio);
   // botanic garden velodyne reordering
   m.def("fill_col_split_row_velodyne", &fill_col_split_row_velodyne);
+
+  m.def("parse_csv_line", &parse_csv_line);
 }
 
 } // namespace evalio
diff --git a/python/evalio/types/base.py b/python/evalio/types/base.py
index c7f02f4e..0841e37d 100644
--- a/python/evalio/types/base.py
+++ b/python/evalio/types/base.py
@@ -12,7 +12,6 @@
 from io import TextIOWrapper
 from typing_extensions import TypeVar
 from evalio.utils import print_warning
-import numpy as np
 import yaml
 
 from pathlib import Path
@@ -20,9 +19,9 @@
 
 from evalio._cpp.types import (  # type: ignore
     SE3,
-    SO3,
     Stamp,
 )
+from evalio._cpp.helpers import parse_csv_line  # type: ignore
 
 from evalio.utils import pascal_to_snake
 
@@ -204,7 +203,7 @@ def from_csv(
         path: Path,
         fieldnames: list[str],
         delimiter: str = ",",
-        skip_lines: Optional[int] = None,
+        skip_lines: int = 0,
     ) -> Trajectory:
         """Flexible loader for stamped poses stored in csv files.
 
@@ -229,41 +228,14 @@ def from_csv(
         poses: list[SE3] = []
         stamps: list[Stamp] = []
 
-        # Pre-check how stamps are stored for efficiency
-        swap_sec_t = "t" in fieldnames
-        has_sec_field = "sec" in fieldnames or "t" in fieldnames
-        has_nsec_field = "nsec" in fieldnames
+        fields = {name: i for i, name in enumerate(fieldnames)}
 
         with open(path) as f:
-            csvfile = list(filter(lambda row: row[0] != "#", f))
-            if skip_lines is not None:
-                csvfile = csvfile[skip_lines:]
-            reader = csv.DictReader(csvfile, fieldnames=fieldnames, delimiter=delimiter)
-            for line in reader:
-                r = SO3(
-                    qw=float(line["qw"]),
-                    qx=float(line["qx"]),
-                    qy=float(line["qy"]),
-                    qz=float(line["qz"]),
-                )
-                t = np.array([float(line["x"]), float(line["y"]), float(line["z"])])
-                pose = SE3(r, t)
-
-                if swap_sec_t:
-                    line["sec"] = line["t"]
-
-                match (has_sec_field, has_nsec_field):
-                    case (True, True):
-                        stamp = Stamp(sec=int(line["sec"]), nsec=int(line["nsec"]))
-                    case (True, False):
-                        # parse separately to get exact stamp
-                        s, ns = line["sec"].split(".")
-                        ns = ns.ljust(9, "0")  # pad to 9 digits for nanoseconds
-                        stamp = Stamp(sec=int(s), nsec=int(ns))
-                    case (False, True):
-                        stamp = Stamp.from_nsec(int(line["nsec"]))
-                    case (False, False):
-                        raise ValueError("Must have at least one of 'sec' or 'nsec'.")
+            csvfile = filter(lambda row: row[0] != "#", f)
+            for i, line in enumerate(csvfile):
+                if i < skip_lines:
+                    continue
+                stamp, pose = parse_csv_line(line, delimiter, fields)
 
                 poses.append(pose)
                 stamps.append(stamp)
diff --git a/tests/test_io.py b/tests/test_io.py
index c6ee043f..251249c0 100644
--- a/tests/test_io.py
+++ b/tests/test_io.py
@@ -1,5 +1,6 @@
 from pathlib import Path
 from evalio import types as ty
+from evalio._cpp.helpers import parse_csv_line  # type: ignore
 import numpy as np
 
 
@@ -73,3 +74,57 @@ def test_trajectory_incremental_serde(tmp_path: Path):
 
     new_traj = ty.Trajectory.from_file(path)
     assert traj == new_traj
+
+
+def test_csv_line():
+    exp_pose = ty.SE3(
+        rot=ty.SO3(
+            qx=-0.9998805501718303,
+            qy=0.005631361428549012,
+            qz=0.0033964292086203895,
+            qw=0.013987044904833533,
+        ),
+        trans=np.array(
+            [
+                0.04846940144357503,
+                -0.03130991015433452,
+                -0.01876146196188756,
+            ]
+        ),
+    )
+
+    fields = ["sec", "x", "y", "z", "qx", "qy", "qz", "qw"]
+    fields = {v: i for i, v in enumerate(fields)}
+
+    # Try a standard one
+    line = "1670403901.143296798,0.04846940144357503,-0.03130991015433452,-0.01876146196188756,-0.9998805501718303,0.005631361428549012,0.0033964292086203895,0.013987044904833533"
+    stamp, pose = parse_csv_line(line, ",", fields)
+
+    assert stamp == ty.Stamp(sec=1670403901, nsec=143296798)
+    assert pose == exp_pose
+
+    # Try one with not padded nsec
+    # Try a standard one
+    line = "1670403901.143296,0.04846940144357503,-0.03130991015433452,-0.01876146196188756,-0.9998805501718303,0.005631361428549012,0.0033964292086203895,0.013987044904833533"
+    stamp, pose = parse_csv_line(line, ",", fields)
+
+    assert stamp == ty.Stamp(sec=1670403901, nsec=143296000)
+    assert pose == exp_pose
+
+    # Try one with both sec and nsec
+    fields = ["sec", "nsec", "x", "y", "z", "qx", "qy", "qz", "qw"]
+    fields = {v: i for i, v in enumerate(fields)}
+    line = "1670403901,143296798,0.04846940144357503,-0.03130991015433452,-0.01876146196188756,-0.9998805501718303,0.005631361428549012,0.0033964292086203895,0.013987044904833533"
+    stamp, pose = parse_csv_line(line, ",", fields)
+
+    assert stamp == ty.Stamp(sec=1670403901, nsec=143296798)
+    assert pose == exp_pose
+
+    # Try one with just nsec
+    fields = ["nsec", "x", "y", "z", "qx", "qy", "qz", "qw"]
+    fields = {v: i for i, v in enumerate(fields)}
+    line = "1670403901143296798,0.04846940144357503,-0.03130991015433452,-0.01876146196188756,-0.9998805501718303,0.005631361428549012,0.0033964292086203895,0.013987044904833533"
+    stamp, pose = parse_csv_line(line, ",", fields)
+
+    assert stamp == ty.Stamp.from_nsec(1670403901143296798)
+    assert pose == exp_pose

From aa0fcce3a84156703e492c7eda8c51925e12f97c Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Fri, 3 Oct 2025 10:29:40 -0400
Subject: [PATCH 29/40] Switch SE3 distance to cpp for speed

---
 cpp/bindings/types.h   |  7 +++++++
 cpp/evalio/types.h     | 19 ++++++++++++-------
 python/evalio/stats.py |  3 +--
 3 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/cpp/bindings/types.h b/cpp/bindings/types.h
index 4fadc7f2..eb175826 100644
--- a/cpp/bindings/types.h
+++ b/cpp/bindings/types.h
@@ -554,6 +554,13 @@ inline void makeTypes(nb::module_& m) {
       "Compute the rotational (degrees) and translational (meters) error "
       "between two SE3s as a tuple (rot, trans)."
     )
+    .def_static(
+      "distance",
+      &SE3::distance,
+      "a"_a,
+      "b"_a,
+      "Compute the distance between two SE3s."
+    )
     .def_static("exp", &SE3::exp, "xi"_a, "Create a SE3 from a 3D vector.")
     .def("log", &SE3::log, "Compute the logarithm of the transformation.")
     .def(nb::self * nb::self, "Compose two rigid body transformations.")
diff --git a/cpp/evalio/types.h b/cpp/evalio/types.h
index e88a02db..f6039eda 100644
--- a/cpp/evalio/types.h
+++ b/cpp/evalio/types.h
@@ -388,13 +388,6 @@ struct SE3 {
     return SE3(inv_rot, inv_rot.rotate(-trans));
   }
 
-  static std::pair<double, double> error(const SE3& a, const SE3& b) {
-    auto delta = a.inverse() * b;
-    double rot_err = delta.rot.log().norm() * (180.0 / M_PI);
-    double trans_err = (delta.trans).norm();
-    return {rot_err, trans_err};
-  }
-
   static SE3 exp(const Eigen::Matrix<double, 6, 1>& xi) {
     Eigen::Vector3d omega = xi.head<3>();
     Eigen::Vector3d xyz = xi.tail<3>();
@@ -463,6 +456,18 @@ struct SE3 {
   bool operator!=(const SE3& other) const {
     return !(*this == other);
   }
+
+  // Helpers for stats computations
+  static std::pair<double, double> error(const SE3& a, const SE3& b) {
+    auto delta = a.inverse() * b;
+    double rot_err = delta.rot.log().norm() * (180.0 / M_PI);
+    double trans_err = (delta.trans).norm();
+    return {rot_err, trans_err};
+  }
+
+  static double distance(const SE3& a, const SE3& b) {
+    return (a.trans - b.trans).norm();
+  }
 };
 
 } // namespace evalio
diff --git a/python/evalio/stats.py b/python/evalio/stats.py
index d3da2b4d..30766a23 100644
--- a/python/evalio/stats.py
+++ b/python/evalio/stats.py
@@ -346,8 +346,7 @@ def rte(
         # Compute deltas for all of ground truth poses
         dist = np.zeros(len_gt)
         for i in range(1, len_gt):
-            diff: NDArray[np.float64] = gt.poses[i].trans - gt.poses[i - 1].trans
-            dist[i] = np.sqrt(diff @ diff)
+            dist[i] = ty.SE3.distance(gt.poses[i], gt.poses[i - 1])
 
         cum_dist = np.cumsum(dist)
         end_idx = 1

From 679301977848fb2c2ab3a4c12ab3f72ede6a3ac9 Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Fri, 3 Oct 2025 10:33:44 -0400
Subject: [PATCH 30/40] Add in copy constructors

---
 cpp/bindings/types.h       | 2 ++
 python/evalio/cli/stats.py | 5 ++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/cpp/bindings/types.h b/cpp/bindings/types.h
index eb175826..defccc39 100644
--- a/cpp/bindings/types.h
+++ b/cpp/bindings/types.h
@@ -80,6 +80,7 @@ inline void makeTypes(nb::module_& m) {
       "nsec"_a,
       "Create a Stamp from seconds and nanoseconds"
     )
+    .def(nb::init<Stamp>(), "other"_a, "Copy constructor for Stamp.")
     .def_static(
       "from_sec",
       &Stamp::from_sec,
@@ -535,6 +536,7 @@ inline void makeTypes(nb::module_& m) {
       "trans"_a,
       "Create a SE3 from a rotation and translation."
     )
+    .def(nb::init<SE3>(), "other"_a, "Copy constructor for SE3.")
     .def_static("identity", &SE3::identity, "Create an identity SE3.")
     .def_static(
       "fromMat",
diff --git a/python/evalio/cli/stats.py b/python/evalio/cli/stats.py
index 359f437a..8af0f059 100644
--- a/python/evalio/cli/stats.py
+++ b/python/evalio/cli/stats.py
@@ -1,4 +1,3 @@
-from copy import copy
 from pathlib import Path
 from typing import Annotated, Any, Callable, Optional, cast
 
@@ -86,8 +85,8 @@ def eval_dataset(
 
         # add metrics
         gt_aligned = Trajectory(
-            stamps=[copy(s) for s in gt_og.stamps],
-            poses=[copy(p) for p in gt_og.poses],
+            stamps=[ty.Stamp(s) for s in gt_og.stamps],
+            poses=[ty.SE3(p) for p in gt_og.poses],
         )
         stats.align(traj, gt_aligned, in_place=True)
         if length is not None and len(traj) > length:

From 9b78587ac83c6619927d93e54d42af72186bc9c8 Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Fri, 3 Oct 2025 10:42:31 -0400
Subject: [PATCH 31/40] Speedup using c yaml loader

---
 python/evalio/types/base.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/python/evalio/types/base.py b/python/evalio/types/base.py
index 0841e37d..f3e8abd0 100644
--- a/python/evalio/types/base.py
+++ b/python/evalio/types/base.py
@@ -103,7 +103,13 @@ def from_yaml(cls, yaml_str: str) -> Metadata | FailedMetadataParse:
         Returns:
             An instance of the metadata class or an error.
         """
-        data = yaml.safe_load(yaml_str)
+        try:
+            data = yaml.load(yaml_str, Loader=yaml.CSafeLoader)
+        except Exception as _:
+            print_warning(
+                "Failed to parse metadata with CSafeLoader, trying SafeLoader"
+            )
+            data = yaml.load(yaml_str, Loader=yaml.SafeLoader)
 
         if "type" not in data:
             return FailedMetadataParse("No type field found in metadata.")

From 936be9b2fbe3ec0bedc541547aa971cf4cf6ccda Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Fri, 3 Oct 2025 13:25:19 -0400
Subject: [PATCH 32/40] Move _check_overstep to cpp

---
 cpp/bindings/ros_pc2.h |  8 ++++++++
 python/evalio/rerun.py |  8 ++++++--
 python/evalio/stats.py | 19 ++++++++++++-------
 3 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/cpp/bindings/ros_pc2.h b/cpp/bindings/ros_pc2.h
index 18839f15..bcd0a8f7 100644
--- a/cpp/bindings/ros_pc2.h
+++ b/cpp/bindings/ros_pc2.h
@@ -460,6 +460,13 @@ inline std::pair<Stamp, SE3> parse_csv_line(
   return std::make_pair(stamp, SE3(r, t));
 }
 
+// Returns False if a is closer to idx, True if b is closer to idx
+inline bool closest(const Stamp& idx, const Stamp& a, const Stamp& b) {
+  auto a_diff = std::abs((a - idx).to_nsec());
+  auto b_diff = std::abs((b - idx).to_nsec());
+  return a_diff > b_diff;
+}
+
 // ---------------------- Create python bindings ---------------------- //
 inline void makeConversions(nb::module_& m) {
   nb::enum_<DataType>(m, "DataType")
@@ -530,6 +537,7 @@ inline void makeConversions(nb::module_& m) {
   m.def("fill_col_split_row_velodyne", &fill_col_split_row_velodyne);
 
   m.def("parse_csv_line", &parse_csv_line);
+  m.def("closest", &closest);
 }
 
 } // namespace evalio
diff --git a/python/evalio/rerun.py b/python/evalio/rerun.py
index 798efd3f..c1d82d1c 100644
--- a/python/evalio/rerun.py
+++ b/python/evalio/rerun.py
@@ -10,7 +10,7 @@
 
 from evalio.datasets import Dataset
 from evalio.pipelines import Pipeline
-from evalio.stats import _check_overstep
+from evalio.stats import closest
 from evalio.types import (
     SE3,
     GroundTruth,
@@ -212,7 +212,11 @@ def log(
                     gt_index = 0
                     while self.gt.stamps[gt_index] < data.stamp:
                         gt_index += 1
-                    if _check_overstep(self.gt.stamps, data.stamp, gt_index):
+                    if not closest(
+                        data.stamp,
+                        self.gt.stamps[gt_index - 1],
+                        self.gt.stamps[gt_index],
+                    ):
                         gt_index -= 1
                     gt_o_T_imu_0 = self.gt.poses[gt_index]
                     self.gt_o_T_imu_o = gt_o_T_imu_0 * imu_o_T_imu_0.inverse()
diff --git a/python/evalio/stats.py b/python/evalio/stats.py
index 30766a23..c654df92 100644
--- a/python/evalio/stats.py
+++ b/python/evalio/stats.py
@@ -2,6 +2,7 @@
 from typing_extensions import TypeVar
 
 from evalio.utils import print_warning
+from evalio._cpp.helpers import closest  # type: ignore
 from . import types as ty
 
 from dataclasses import dataclass
@@ -14,10 +15,6 @@
 from copy import deepcopy
 
 
-def _check_overstep(stamps: list[ty.Stamp], s: ty.Stamp, idx: int) -> bool:
-    return abs((stamps[idx - 1] - s).to_sec()) < abs((stamps[idx] - s).to_sec())
-
-
 class MetricKind(StrEnum):
     """Simple enum to define the metric to use for summarizing the error. Used in [Error][evalio.stats.Error.summarize]."""
 
@@ -179,7 +176,11 @@ def align_stamps(traj1: ty.Trajectory[M1], traj2: ty.Trajectory[M2]):
     first_pose_idx = 0
     while traj1.stamps[first_pose_idx] < traj2.stamps[0]:
         first_pose_idx += 1
-    if _check_overstep(traj1.stamps, traj2.stamps[0], first_pose_idx):
+    if not closest(
+        traj2.stamps[0],
+        traj1.stamps[first_pose_idx - 1],
+        traj1.stamps[first_pose_idx],
+    ):
         first_pose_idx -= 1
     traj1.stamps = traj1.stamps[first_pose_idx:]
     traj1.poses = traj1.poses[first_pose_idx:]
@@ -188,7 +189,11 @@ def align_stamps(traj1: ty.Trajectory[M1], traj2: ty.Trajectory[M2]):
     first_pose_idx = 0
     while traj2.stamps[first_pose_idx] < traj1.stamps[0]:
         first_pose_idx += 1
-    if _check_overstep(traj2.stamps, traj1.stamps[0], first_pose_idx):
+    if not closest(
+        traj1.stamps[0],
+        traj2.stamps[first_pose_idx - 1],
+        traj2.stamps[first_pose_idx],
+    ):
         first_pose_idx -= 1
     traj2.stamps = traj2.stamps[first_pose_idx:]
     traj2.poses = traj2.poses[first_pose_idx:]
@@ -214,7 +219,7 @@ def align_stamps(traj1: ty.Trajectory[M1], traj2: ty.Trajectory[M2]):
             traj1_idx += 1
 
         # go back one if we overshot
-        if _check_overstep(traj1.stamps, stamp, traj1_idx):
+        if not closest(stamp, traj1.stamps[traj1_idx - 1], traj1.stamps[traj1_idx]):
             traj1_idx -= 1
 
         traj1_stamps.append(traj1.stamps[traj1_idx])

From c18bbf269826ed390ead985342f0db45d8ea1c29 Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Fri, 3 Oct 2025 15:17:44 -0400
Subject: [PATCH 33/40] Some niceties for output in stats

---
 python/evalio/cli/stats.py             | 5 ++++-
 python/evalio/datasets/multi_campus.py | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/python/evalio/cli/stats.py b/python/evalio/cli/stats.py
index 8af0f059..c040a9f5 100644
--- a/python/evalio/cli/stats.py
+++ b/python/evalio/cli/stats.py
@@ -337,6 +337,9 @@ def evaluate_typer(
         ((pl.col("sequence_length") / pl.col("total_elapsed")).alias("hz"))
     )
 
+    # rename length for brevity
+    df = df.rename({"sequence_length": "len"})
+
     # print columns if requested
     if print_columns:
         c.print("Available columns:")
@@ -399,7 +402,7 @@ def evaluate_typer(
 
     # ------------------------- Print ------------------------- #
     # Print sequence by sequence
-    for sequence in df["sequence"].unique():
+    for sequence in sorted(df["sequence"].unique()):
         df_sequence = df.filter(pl.col("sequence") == sequence)
         df_sequence = df_sequence.drop("sequence")
         if df_sequence.is_empty():
diff --git a/python/evalio/datasets/multi_campus.py b/python/evalio/datasets/multi_campus.py
index 43534bfd..517ef332 100644
--- a/python/evalio/datasets/multi_campus.py
+++ b/python/evalio/datasets/multi_campus.py
@@ -79,7 +79,7 @@ def data_iter(self) -> DatasetIterator:
     def ground_truth_raw(self) -> Trajectory:
         return Trajectory.from_csv(
             self.folder / "pose_inW.csv",
-            ["num", "t", "x", "y", "z", "qx", "qy", "qz", "qw"],
+            ["num", "sec", "x", "y", "z", "qx", "qy", "qz", "qw"],
             skip_lines=1,
         )
 

From 21a1419325faf8dd5d8dc1c2c591e830dcc5275c Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Fri, 3 Oct 2025 17:23:29 -0400
Subject: [PATCH 34/40] Bump rerun to 0.25

---
 pyproject.toml         |  2 +-
 python/evalio/rerun.py |  2 +-
 uv.lock                | 14 +++++++-------
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index d7929985..f27a82dd 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -29,7 +29,7 @@ keywords = [
 license = { file = "LICENSE.txt" }
 
 [project.optional-dependencies]
-vis = ["rerun-sdk>=0.23"]
+vis = ["rerun-sdk>=0.25"]
 
 [build-system]
 requires = ["scikit-build-core>=0.8", "nanobind>=2.9.2", "numpy"]
diff --git a/python/evalio/rerun.py b/python/evalio/rerun.py
index c1d82d1c..ae420069 100644
--- a/python/evalio/rerun.py
+++ b/python/evalio/rerun.py
@@ -118,7 +118,7 @@ def __init__(self, args: VisArgs, pipeline_names: list[str]):
                 + [skybox_light_rgb(dir) for dir in directions]
             )
 
-        def _blueprint(self) -> rr.BlueprintLike:
+        def _blueprint(self) -> rrb.BlueprintLike:
             # Eventually we'll be able to glob these, but for now, just take in the names beforehand
             # https://github.com/rerun-io/rerun/issues/6673
             # Once this is closed, we'll be able to remove pipelines as a parameter here and in new_recording
diff --git a/uv.lock b/uv.lock
index da07a255..17154e60 100644
--- a/uv.lock
+++ b/uv.lock
@@ -333,7 +333,7 @@ requires-dist = [
     { name = "polars", specifier = ">=1.33.1" },
     { name = "pyyaml", specifier = ">=6.0" },
     { name = "rapidfuzz", specifier = ">=3.12.2" },
-    { name = "rerun-sdk", marker = "extra == 'vis'", specifier = ">=0.23" },
+    { name = "rerun-sdk", marker = "extra == 'vis'", specifier = ">=0.25" },
     { name = "rosbags", specifier = ">=0.10.10" },
     { name = "tqdm", specifier = ">=4.66" },
     { name = "typer", specifier = ">=0.15.3" },
@@ -1355,7 +1355,7 @@ socks = [
 
 [[package]]
 name = "rerun-sdk"
-version = "0.23.1"
+version = "0.25.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "attrs" },
@@ -1365,11 +1365,11 @@ dependencies = [
     { name = "typing-extensions" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/dd/6e/a125f4fe2de3269f443b7cb65d465ffd37a836a2dac7e4318e21239d78c8/rerun_sdk-0.23.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:fe06d21cfcf4d84a9396f421d4779efabec7e9674d232a2c552c8a91d871c375", size = 66094053, upload-time = "2025-04-25T13:15:48.669Z" },
-    { url = "https://files.pythonhosted.org/packages/55/f6/b6d13322b05dc77bd9a0127e98155c2b7ee987a236fd4d331eed2e547a90/rerun_sdk-0.23.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:823ae87bfa644e06fb70bada08a83690dd23d9824a013947f80a22c6731bdc0d", size = 62047843, upload-time = "2025-04-25T13:15:54.48Z" },
-    { url = "https://files.pythonhosted.org/packages/a5/7f/6a7422cb727e14a65b55b0089988eeea8d0532c429397a863e6ba395554a/rerun_sdk-0.23.1-cp39-abi3-manylinux_2_31_aarch64.whl", hash = "sha256:dc5129f8744f71249bf45558c853422c51ef39b6b5eea0ea1f602c6049ce732f", size = 68214509, upload-time = "2025-04-25T13:15:59.339Z" },
-    { url = "https://files.pythonhosted.org/packages/4f/86/3aee9eadbfe55188a2c7d739378545b4319772a4d3b165e8d3fc598fa630/rerun_sdk-0.23.1-cp39-abi3-manylinux_2_31_x86_64.whl", hash = "sha256:ee0d0e17df0e08be13b77cc74884c5d8ba8edb39b6f5a60dc2429d39033d90f6", size = 71442196, upload-time = "2025-04-25T13:16:04.405Z" },
-    { url = "https://files.pythonhosted.org/packages/a7/ba/028bd382e2ae21e6643cec25f423285dbc6b328ce56d55727b4101ef9443/rerun_sdk-0.23.1-cp39-abi3-win_amd64.whl", hash = "sha256:d4273db55b56310b053a2de6bf5927a8692cf65f4d234c6e6928fb24ed8a960d", size = 57583198, upload-time = "2025-04-25T13:16:08.905Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/e7/50731172bf2e6fe1f15e22a081336d144451d5873c044b440a268a772dae/rerun_sdk-0.25.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:41eec7d7ea7c048fc475ccd128ff2522dd3aac49daf5c9db50e8ed63ddc05583", size = 88331558, upload-time = "2025-09-19T09:32:22.498Z" },
+    { url = "https://files.pythonhosted.org/packages/42/3e/7811b97a06a3edfc606463a287eb560b3e3cb7bc32ccd861adc7e0d511c7/rerun_sdk-0.25.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:4f75960a89949b90f443b814c24647e99ee63d2662f53ee5b97744e25f1d85c0", size = 82275113, upload-time = "2025-09-19T09:32:27.263Z" },
+    { url = "https://files.pythonhosted.org/packages/07/15/3c9c60b28c0e399f980e58df6d7a98e82890623f99b39c731c6437fa86b5/rerun_sdk-0.25.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2880104561d8719edfe605e636d4bd00ef0f7656fde0af869feb00cf9c1e8d27", size = 90753199, upload-time = "2025-09-19T09:32:31.101Z" },
+    { url = "https://files.pythonhosted.org/packages/52/b3/05c25a8ae701b71b2bb5f61101bdd730ffb8d1027537de5ad97123b99735/rerun_sdk-0.25.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:000c51f8b0faa3ed33800c62ed95249c527152e9615503ce8a81787acefd2361", size = 95201637, upload-time = "2025-09-19T09:32:35.755Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/34/1cd4cee6bace649e68b04c350f8a8ea97de339439cb01832c6d33560532e/rerun_sdk-0.25.1-cp39-abi3-win_amd64.whl", hash = "sha256:f884f5ada4581e4f50448ef641e46c609e747a200a64059de656fb4e4b10cff9", size = 76612241, upload-time = "2025-09-19T09:32:40.97Z" },
 ]
 
 [[package]]

From 2ce5f6db7511a84088fa8381ff5d843505bec019 Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Mon, 13 Oct 2025 10:00:57 -0400
Subject: [PATCH 35/40] Some misc cleanups throughout

---
 python/evalio/cli/run.py        | 10 +++++++++-
 python/evalio/cli/stats.py      | 14 +++++++-------
 python/evalio/types/base.py     | 14 ++++++++------
 python/evalio/types/extended.py | 23 +++++++++++++++++++++++
 4 files changed, 47 insertions(+), 14 deletions(-)

diff --git a/python/evalio/cli/run.py b/python/evalio/cli/run.py
index d55e0708..bb29282a 100644
--- a/python/evalio/cli/run.py
+++ b/python/evalio/cli/run.py
@@ -117,7 +117,15 @@ def run_from_cli(
     if config is not None:
         # load from yaml
         with open(config, "r") as f:
-            params = yaml.safe_load(f)
+            try:
+                Loader = yaml.CSafeLoader
+            except Exception as _:
+                print_warning(
+                    "Failed to import yaml.CSafeLoader, trying yaml.SafeLoader"
+                )
+                Loader = yaml.SafeLoader
+
+            params = yaml.load(f, Loader=Loader)
 
         if "datasets" not in params:
             raise typer.BadParameter(
diff --git a/python/evalio/cli/stats.py b/python/evalio/cli/stats.py
index c040a9f5..5c63576d 100644
--- a/python/evalio/cli/stats.py
+++ b/python/evalio/cli/stats.py
@@ -82,6 +82,9 @@ def eval_dataset(
     results: list[dict[str, Any]] = []
     for index, traj in enumerate(all_trajs):
         r: dict[str, Any] = {}
+        hz = None
+        if traj.metadata.total_elapsed is not None:
+            hz = len(traj) / traj.metadata.total_elapsed
 
         # add metrics
         gt_aligned = Trajectory(
@@ -104,6 +107,8 @@ def eval_dataset(
 
         # add metadata
         r |= traj.metadata.to_dict()
+        # add extra hz field
+        r["hz"] = hz
         # flatten pipeline params
         r.update(r["pipeline_params"])
         del r["pipeline_params"]
@@ -129,7 +134,7 @@ def _contains_dir(directory: Path) -> bool:
 def evaluate(
     directories: list[Path],
     windows: list[stats.WindowKind],
-    metric: stats.MetricKind,
+    metric: stats.MetricKind = stats.MetricKind.sse,
     length: Optional[int] = None,
     visualize: bool = False,
 ) -> list[dict[str, Any]]:
@@ -168,7 +173,7 @@ def evaluate_typer(
     sort: Annotated[
         Optional[str],
         typer.Option(
-            "-s",
+            "-S",
             "--sort",
             help="Sort results by the name of a column. Defaults to RTEt.",
             rich_help_panel="Output options",
@@ -332,11 +337,6 @@ def evaluate_typer(
 
     df = pl.DataFrame(results)
 
-    # clean up timing
-    df = df.with_columns(
-        ((pl.col("sequence_length") / pl.col("total_elapsed")).alias("hz"))
-    )
-
     # rename length for brevity
     df = df.rename({"sequence_length": "len"})
 
diff --git a/python/evalio/types/base.py b/python/evalio/types/base.py
index f3e8abd0..48283936 100644
--- a/python/evalio/types/base.py
+++ b/python/evalio/types/base.py
@@ -104,14 +104,16 @@ def from_yaml(cls, yaml_str: str) -> Metadata | FailedMetadataParse:
             An instance of the metadata class or an error.
         """
         try:
-            data = yaml.load(yaml_str, Loader=yaml.CSafeLoader)
+            Loader = yaml.CSafeLoader
         except Exception as _:
-            print_warning(
-                "Failed to parse metadata with CSafeLoader, trying SafeLoader"
-            )
-            data = yaml.load(yaml_str, Loader=yaml.SafeLoader)
+            print_warning("Failed to import yaml.CSafeLoader, trying yaml.SafeLoader")
+            Loader = yaml.SafeLoader
+
+        data = yaml.load(yaml_str, Loader=Loader)
 
-        if "type" not in data:
+        if data is None:
+            return FailedMetadataParse("Metadata failed to parse.")
+        elif "type" not in data:
             return FailedMetadataParse("No type field found in metadata.")
 
         for name, subclass in cls._registry.items():
diff --git a/python/evalio/types/extended.py b/python/evalio/types/extended.py
index ffb0f54b..68226af3 100644
--- a/python/evalio/types/extended.py
+++ b/python/evalio/types/extended.py
@@ -68,6 +68,29 @@ def from_dict(cls, data: dict[str, Any]) -> Self:
 
         return super().from_dict(data)
 
+    @classmethod
+    def from_pl_ds(
+        cls, pipe: type[pl.Pipeline], ds_obj: ds.Dataset, **kwargs: Any
+    ) -> Self:
+        """Create an Experiment from a pipeline and dataset.
+
+        Args:
+            pipe (type[pl.Pipeline]): The pipeline class.
+            ds_obj (ds.Dataset): The dataset object.
+            **kwargs: Additional keyword arguments to pass to the Experiment constructor.
+
+        Returns:
+            Self: The created Experiment instance.
+        """
+        return cls(
+            name=pipe.name(),
+            sequence=ds_obj,
+            sequence_length=len(ds_obj),
+            pipeline=pipe,
+            pipeline_version=pipe.version(),
+            pipeline_params=pipe.default_params() | kwargs,
+        )
+
     def setup(
         self,
     ) -> tuple[pl.Pipeline, ds.Dataset] | ds.SequenceNotFound | pl.PipelineNotFound:

From e0d68ba956b60ee4259a2c7e081c8b82019b6a7b Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Tue, 14 Oct 2025 20:41:23 -0400
Subject: [PATCH 36/40] Update python/evalio/stats.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
---
 python/evalio/stats.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/python/evalio/stats.py b/python/evalio/stats.py
index c654df92..d681ffe7 100644
--- a/python/evalio/stats.py
+++ b/python/evalio/stats.py
@@ -345,7 +345,6 @@ def rte(
             window_deltas_poses.append(traj.poses[i].inverse() * traj.poses[end_idx])
             window_deltas_gts.append(gt.poses[i].inverse() * gt.poses[end_idx])
 
-            end_idx_prev = end_idx
 
     elif isinstance(window, WindowMeters):
         # Compute deltas for all of ground truth poses

From e9dad22e3e24a2a91c3dba99382d29ac61f08c06 Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Tue, 14 Oct 2025 20:41:32 -0400
Subject: [PATCH 37/40] Update python/evalio/rerun.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
---
 python/evalio/rerun.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/python/evalio/rerun.py b/python/evalio/rerun.py
index ae420069..1b765e8f 100644
--- a/python/evalio/rerun.py
+++ b/python/evalio/rerun.py
@@ -10,7 +10,6 @@
 
 from evalio.datasets import Dataset
 from evalio.pipelines import Pipeline
-from evalio.stats import closest
 from evalio.types import (
     SE3,
     GroundTruth,

From 93dbc3886236b8ffaf657778def68b6e84364aba Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Tue, 14 Oct 2025 20:41:46 -0400
Subject: [PATCH 38/40] Update python/evalio/datasets/base.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
---
 python/evalio/datasets/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/evalio/datasets/base.py b/python/evalio/datasets/base.py
index 157aed38..24d0628d 100644
--- a/python/evalio/datasets/base.py
+++ b/python/evalio/datasets/base.py
@@ -241,7 +241,7 @@ def _warn_default_dir(cls):
         global _DATA_DIR, _WARNED
         if not _WARNED and _DATA_DIR == Path("./evalio_data"):
             print_warning(
-                "Using default './evalio_data' for base data directory. Override by setting [magenta]EVALIO_DATA[/magenta], [magenta]evalio.set_DATA_DIR(path)[/magenta] in python, or [magenta]-D[/magenta] in the CLI."
+                "Using default './evalio_data' for base data directory. Override by setting [magenta]EVALIO_DATA[/magenta], [magenta]evalio.set_data_dir(path)[/magenta] in python, or [magenta]-D[/magenta] in the CLI."
             )
             _WARNED = True
 

From 9d5aaae6f10c74e88bfda3c9249251a478c8a9c6 Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Tue, 14 Oct 2025 20:49:50 -0400
Subject: [PATCH 39/40] Fix some pyproject deprecations

---
 pyproject.toml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index f27a82dd..953352f3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -63,8 +63,8 @@ build-verbosity = 1
 MACOSX_DEPLOYMENT_TARGET = "11"
 
 # local development
-[tool.uv]
-dev-dependencies = [
+[dependency-groups]
+dev = [
     "cmake<4.0.0",
     "compdb>=0.2.0",
     "ruff>=0.6.8",
@@ -87,7 +87,7 @@ dev-dependencies = [
 
 [tool.ruff]
 exclude = ["cpp/**/*"]
-ignore = ["E731"]
+lint.ignore = ["E731"]
 
 [tool.ruff.format]
 # https://docs.astral.sh/ruff/configuration/

From 2bd3eec6209c6354b4d84278405f91df04e79fa0 Mon Sep 17 00:00:00 2001
From: Easton Potokar <contagon6@gmail.com>
Date: Tue, 14 Oct 2025 20:52:57 -0400
Subject: [PATCH 40/40] Fix rerun import

---
 python/evalio/rerun.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/python/evalio/rerun.py b/python/evalio/rerun.py
index 1b765e8f..406b4f75 100644
--- a/python/evalio/rerun.py
+++ b/python/evalio/rerun.py
@@ -21,6 +21,7 @@
     Trajectory,
 )
 from evalio.utils import print_warning
+from evalio._cpp.helpers import closest  # type: ignore
 
 
 # These colors are pulled directly from the rerun skybox colors