Add Python visualization notebooks

.gitignore

@@ -10,3 +10,5 @@ docs/_build/
 docs/api/reference/
 docs/xml/
 __pycache__
+*.egg-info/
+.pytest_cache/

CMakeLists.txt

@@ -51,6 +51,9 @@ option(GridKit_ENABLE_THREADS "Enable THreads" OFF)
 # ZeroMQ support is disabled by default
 option(GridKit_ENABLE_ZMQ "Enable ZeroMQ support" OFF)
 
+# Python notebooks under application/notebooks are disabled by default
+option(GridKit_ENABLE_NOTEBOOKS "Include the Python notebooks under application/notebooks" OFF)
+
 # This allows use of "GRIDKIT_*" versions of the above options
 list(
   APPEND
@@ -63,7 +66,8 @@ list(
   UBSAN
   OPENMP
   THREADS
-  ZMQ)
+  ZMQ
+  NOTEBOOKS)
 foreach(_opt IN LISTS _gridkit_enable_options)
   if(NOT DEFINED GRIDKIT_ENABLE_${_opt})
     set(GRIDKIT_ENABLE_${_opt} ${GridKit_ENABLE_${_opt}})

INSTALL.md

@@ -114,6 +114,7 @@ installed headers, libraries, and CMake config files.
 | `GridKit_ENABLE_ENZYME` | `OFF` | Build with Enzyme automatic differentiation |
 | `GridKit_ENABLE_ASAN` | `OFF` | Enable address sanitizer |
 | `GridKit_ENABLE_UBSAN` | `OFF` | Enable undefined behavior sanitizer |
+| `GridKit_ENABLE_NOTEBOOKS` | `OFF` | Include the Python notebooks under `application/notebooks` |
 | `BUILD_SHARED_LIBS` | `ON` | Build shared libraries |
 | `CMAKE_INSTALL_PREFIX` | system default | Installation root |
 

application/CMakeLists.txt

@@ -1,3 +1,7 @@
 if(TARGET SUNDIALS::idas)
   add_subdirectory(PhasorDynamics)
 endif()
+
+if(GridKit_ENABLE_NOTEBOOKS)
+  add_subdirectory(notebooks)
+endif()

application/notebooks/CMakeLists.txt

@@ -0,0 +1,23 @@
+option(GridKit_ENABLE_NOTEBOOKS_PYENV
+       "Create a venv and install application/notebooks Python dependencies" OFF)
+
+find_package(Python3 COMPONENTS Interpreter)
+
+if(Python3_FOUND)
+  set(_nb_venv ${CMAKE_CURRENT_BINARY_DIR}/notebooks-venv)
+
+  add_custom_target(
+    notebooks_pyenv
+    COMMAND ${Python3_EXECUTABLE} -m venv ${_nb_venv}
+    COMMAND ${_nb_venv}/bin/pip install --upgrade pip
+    COMMAND ${_nb_venv}/bin/pip install -e ${CMAKE_CURRENT_SOURCE_DIR}[notebook,test]
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMENT "Creating venv and installing application/notebooks Python dependencies"
+    VERBATIM)
+
+  if(GridKit_ENABLE_NOTEBOOKS_PYENV)
+    add_custom_target(notebooks_pyenv_all ALL DEPENDS notebooks_pyenv)
+  endif()
+else()
+  message(STATUS "Python3 interpreter not found; notebooks_pyenv target unavailable")
+endif()

application/notebooks/README.md

@@ -0,0 +1,90 @@
+# application/notebooks
+
+Python notebooks and supporting packages for exploratory analysis and
+visualization of GridKit cases. Not part of the C++ build; enabled via the
+`GridKit_ENABLE_NOTEBOOKS` CMake option (see below) or usable standalone
+with `pip`.
+
+## Packages
+
+| Package | Purpose |
+|---|---|
+| `m_viz/` | MATPOWER `.m` case parsing, geo-coordinate attachment, and Plotly geo plotting. Used by `m_viz.ipynb`. |
+| `gridkit_common/` | Small helpers shared across notebooks (linking MATPOWER tables back to GridKit JSON case IDs). |
+| `uq_sampling/` | Parameter sampling (Latin Hypercube / random), run-directory orchestration, and Genrou dispatch patching for uncertainty-quantification sweeps. Not yet exercised by a notebook in this repo — added ahead of the UQ sweep notebook that will use it. |
+
+## Environment setup
+
+Developed and tested with Python 3.12; 3.11+ should also work.
+
+```sh
+conda create --prefix ./my-env python=3.12 --yes
+conda activate ./my-env
+pip install -e .[notebook,test]
+```
+
+Or with a plain venv:
+
+```sh
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -e .[notebook,test]
+```
+
+This installs `m_viz`, `gridkit_common`, and `uq_sampling` as regular
+importable packages — no `sys.path` manipulation needed in notebooks.
+
+## Notebooks
+
+### m_viz
+
+`m_viz.ipynb` parses MATPOWER `.m` case files (ACOPF or PF solutions) into normalized pandas
+DataFrames and produces interactive Plotly geo maps. Geographic coordinates come from a `.gic`
+or `.AUX` file. The map shows bus loads (PD), dispatched generation (fuel-colored, sized by MW),
+and branch loading percentage (viridis color scale).
+
+Optionally, set `GRIDKIT_REPO` to the root of a local GridKit repository clone to augment hover
+labels with GridKit-assigned bus and branch IDs (`gridkit_common.ids`, requires `scipy`). Set
+`GRIDKIT_REPO = None` to skip this.
+
+Supported cases: **Hawaii40**, **Illinois (ACTIVSg200)**, **Texas (ACTIVSg2000)**, **WECC (ACTIVSg10k)**.
+
+#### Input data
+
+Download TAMU synthetic grid cases from
+[Texas A&M Electric Grid Test Cases](https://electricgrids.engr.tamu.edu/electric-grid-test-cases/).
+After unzipping, set `CASE_DATA_DIR` (or the `GRIDKIT_CASE_DATA_DIR` environment variable) to the
+folder containing the extracted files and `CASE_NAME` to match. Everything else is auto-detected.
+
+##### Hawaii40
+
+| File | Used for |
+|---|---|
+| `Hawaii40_20231026.m` | MATPOWER case (buses, generators, branches) |
+| `Hawaii40_GIC_data.gic` | Geographic coordinates (preferred) |
+| `Hawaii40_20231026.AUX` | Geographic coordinates (fallback if no `.gic`) |
+
+##### Illinois (ACTIVSg200)
+
+| File | Used for |
+|---|---|
+| `case_ACTIVSg200.m` | MATPOWER case (buses, generators, branches) |
+| `ACTIVSg200_GIC_data.gic` | Geographic coordinates (preferred) |
+| `ACTIVSg200.AUX` | Geographic coordinates (fallback if no `.gic`) |
+
+##### Texas (ACTIVSg2000)
+
+| File | Used for |
+|---|---|
+| `case_ACTIVSg2000.m` | MATPOWER case (buses, generators, branches) |
+| `ACTIVSg2000_GIC_data.gic` | Geographic coordinates (preferred) |
+| `ACTIVSg2000_dynamics.AUX` | Geographic coordinates (fallback if no `.gic`) |
+
+##### WECC (ACTIVSg10k)
+
+| File | Used for |
+|---|---|
+| `case_ACTIVSg10k.m` | MATPOWER case (buses, generators, branches) |
+| `ACTIVSg10k_GIC_data.gic` | Geographic coordinates |
+
+More notebooks will be added in future PRs.

application/notebooks/gridkit_common/__init__.py

@@ -0,0 +1,5 @@
+"""Small helpers shared across application/notebooks notebooks."""
+
+from .ids import attach_json_ids, get_case_path_for_editing
+
+__all__ = ["attach_json_ids", "get_case_path_for_editing"]

application/notebooks/gridkit_common/ids.py

@@ -0,0 +1,83 @@
+"""Helpers linking MATPOWER case tables back to GridKit JSON case files."""
+
+from __future__ import annotations
+
+import json
+import os
+
+
+def attach_json_ids(case_data, json_path) -> None:
+    """Attach GridKit JSON identifiers to case_data dataframes in-place.
+
+    Adds columns:
+      - bus_df:    ``json_bus_number`` (int, same as BUS_I — confirms correspondence)
+      - gen_df:    ``json_gen_id`` (str like "2_1"; None for offline/absent gens)
+      - branch_df: ``json_branch_id`` (str like "BR_1_2_1")
+
+    Parameters
+    ----------
+    case_data : m_viz.MatpowerCaseData
+        Object whose ``.bus``, ``.gen``, ``.branch`` DataFrames will be mutated.
+    json_path : str or Path
+        Path to a GridKit JSON case file (hawaii.json style).
+    """
+    with open(json_path) as fh:
+        case = json.load(fh)
+
+    # --- bus: json number == BUS_I (trivially identical, added for hover clarity) ---
+    case_data.bus["json_bus_number"] = case_data.bus["BUS_I"].astype(int)
+
+    # --- generators: derive json_gen_id from rank within GEN_BUS group ---
+    # Build set of ids that actually exist in the JSON (for validation / offline marking)
+    json_gen_ids = {d["id"] for d in case["devices"] if d.get("class") == "Genrou"}
+
+    gen_df = case_data.gen
+    has_status = "GEN_STATUS" in gen_df.columns
+
+    json_gen_id_col = []
+    # rank counter per bus
+    rank: dict[int, int] = {}
+    for row in gen_df.itertuples():
+        bus = int(row.GEN_BUS)
+        rank[bus] = rank.get(bus, 0) + 1
+        candidate = f"{bus}_{rank[bus]}"
+        # Mark as None if offline in .m (GEN_STATUS=0) or absent from JSON
+        if has_status and int(getattr(row, "GEN_STATUS", 1)) == 0:
+            json_gen_id_col.append(None)
+        elif candidate not in json_gen_ids:
+            # absent from JSON even though status=1 — mark None
+            json_gen_id_col.append(None)
+        else:
+            json_gen_id_col.append(candidate)
+
+    case_data.gen["json_gen_id"] = json_gen_id_col
+
+    # --- gen: also store the JSON bus "number" for each gen (== GEN_BUS == BUS_I) ---
+    # Only set when the gen is present in JSON; None for offline/absent gens.
+    case_data.gen["json_gen_bus_number"] = [
+        int(row.GEN_BUS) if gid is not None else None
+        for row, gid in zip(gen_df.itertuples(), json_gen_id_col)
+    ]
+
+    # --- branches: derive json_branch_id from (F_BUS, T_BUS) rank ---
+    parallel_count: dict[tuple, int] = {}
+    json_branch_id_col = []
+    for row in case_data.branch.itertuples():
+        key = (int(row.F_BUS), int(row.T_BUS))
+        parallel_count[key] = parallel_count.get(key, 0) + 1
+        json_branch_id_col.append(f"BR_{key[0]}_{key[1]}_{parallel_count[key]}")
+
+    case_data.branch["json_branch_id"] = json_branch_id_col
+
+
+def get_case_path_for_editing(test_run_dir, example_dir, case_name):
+    """
+    Resolve case path for edits.
+
+    Prefer the copied case file in test_run_dir. If it does not exist,
+    fall back to the source/example directory.
+    """
+    run_case = os.path.join(test_run_dir, case_name)
+    if os.path.exists(run_case):
+        return run_case
+    return os.path.join(example_dir, case_name)