From c377513f1d07da6a62fc434948a5c6dec7984384 Mon Sep 17 00:00:00 2001 From: jepegit Date: Sun, 26 Jul 2026 21:05:19 +0200 Subject: [PATCH 1/2] feat(batch): batch v3 package foundation - journal model + layout (#698) Start the batch v3 redesign (Epic A, #696) as a new cellpy/batch/ package built alongside the old utils/batch_tools. - layout.py: BatchPaths (pure, immutable path computation) + ensure_dirs() as the ONLY mkdir - splits the concern the legacy paginate() conflated (dumpers created folders as a side effect of exporting). - journal.py: Journal model with polars pages (keys-in-columns: the cell label is a `filename` column, never an index) + read_journal/write_journal round-tripping the compatible info_df/metadata/session JSON format, and journal_from_frame. raw_file_names is normalised to List[str], removing the legacy str-or-list ambiguity. Follow-ups within #698: custom-JSON reader (folds #345), Excel read-only, legacy old-shape loader. Co-Authored-By: Claude Opus 4.8 --- cellpy/batch/__init__.py | 32 +++++++ cellpy/batch/journal.py | 177 +++++++++++++++++++++++++++++++++++++++ cellpy/batch/layout.py | 74 ++++++++++++++++ tests/test_batch_v3.py | 94 +++++++++++++++++++++ 4 files changed, 377 insertions(+) create mode 100644 cellpy/batch/__init__.py create mode 100644 cellpy/batch/journal.py create mode 100644 cellpy/batch/layout.py create mode 100644 tests/test_batch_v3.py diff --git a/cellpy/batch/__init__.py b/cellpy/batch/__init__.py new file mode 100644 index 00000000..f7d0ca77 --- /dev/null +++ b/cellpy/batch/__init__.py @@ -0,0 +1,32 @@ +"""cellpy.batch -- the batch v3 subsystem (#696). + +A boring, standard architecture for batch processing, replacing the +``utils/batch_tools`` "farm/barn" machinery. Built alongside the old code; +``cellpy.utils.batch`` becomes a thin re-export/shim. + +Modules land incrementally (plan sections 4 & 6): + journal -- Journal model + json readers/writers (#698, this arc) + layout -- BatchPaths: pure path computation + ensure_dirs (#698) + policy -- LoadPolicy / CellSpec typed options (#699) + runner -- load_cell / run -> BatchResult (#700) + ... +""" + +from __future__ import annotations + +from cellpy.batch.journal import ( + Journal, + journal_from_frame, + read_journal, + write_journal, +) +from cellpy.batch.layout import BatchPaths, ensure_dirs + +__all__ = [ + "Journal", + "read_journal", + "write_journal", + "journal_from_frame", + "BatchPaths", + "ensure_dirs", +] diff --git a/cellpy/batch/journal.py b/cellpy/batch/journal.py new file mode 100644 index 00000000..70f3d775 --- /dev/null +++ b/cellpy/batch/journal.py @@ -0,0 +1,177 @@ +"""Batch journal model + JSON IO (batch v3, #698). + +A journal is a *document*, not an actor: reading one never touches the +filesystem layout, and the data model is separated from serialisation. This is +the successor of ``utils/batch_tools/batch_journals.LabJournal`` (a ~1100-line +class mixing data model, three file formats, path fixing, selection state and +folder generation). Folder layout lives in :mod:`cellpy.batch.layout`. + +The on-disk JSON format is preserved for compatibility: a top-level object with +``info_df`` (pages, pandas ``to_json`` "columns" orient), ``metadata`` and +``session``. Pages follow the keys-in-columns law (polars report section 1.3): +the cell label lives in the ``filename`` *column*, never in an index. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path + +import pandas as pd +import polars as pl + +from cellpy.parameters.internal_settings import ( + get_headers_journal, + keys_journal_session, +) + +_hdr = get_headers_journal() +#: Name of the column holding the (unique) cell label. +FILENAME = _hdr["filename"] + +#: Bump when the on-disk journal shape changes in a non-backward way. +JOURNAL_FORMAT_VERSION = 1 + + +def _empty_session() -> dict: + return {key: None for key in keys_journal_session} + + +@dataclass +class Journal: + """A batch journal: the cells of an experiment plus session/meta state. + + Attributes: + name: batch name. + project: project name. + pages: one row per cell; ``filename`` is a column (keys-in-columns). + session: mutable session state (starred/bad_cells/bad_cycles/notes). + meta: free-form metadata carried through save/load (name, project, + time_stamp, project_dir, ...). + """ + + name: str | None = None + project: str | None = None + pages: pl.DataFrame = field(default_factory=pl.DataFrame) + session: dict = field(default_factory=_empty_session) + meta: dict = field(default_factory=dict) + + @property + def cell_names(self) -> list[str]: + """The cell labels, in page order.""" + if FILENAME in self.pages.columns: + return self.pages[FILENAME].to_list() + return [] + + def __len__(self) -> int: + return self.pages.height + + +def _to_polars(pdf: pd.DataFrame) -> pl.DataFrame: + """Convert a pandas pages frame to polars, column by column. + + Columns that hold any python ``list`` (e.g. ``raw_file_names``, where a + cell may have several raw files) are normalised to a ``List[str]`` column: + scalar values are wrapped in a single-element list and nulls become empty + lists. This removes the legacy str-or-list ambiguity that made pages + impossible to represent as a typed frame. + """ + data: dict[str, pl.Series] = {} + for col in pdf.columns: + values = pdf[col].tolist() + if any(isinstance(v, list) for v in values): + normalised = [ + v + if isinstance(v, list) + else ([] if v is None or (isinstance(v, float) and pd.isna(v)) else [v]) + for v in values + ] + data[col] = pl.Series(col, normalised, dtype=pl.List(pl.Utf8)) + else: + data[col] = pl.Series(col, values) + return pl.DataFrame(data) + + +def _pages_from_info_df(info_df: dict) -> pl.DataFrame: + """Parse the legacy ``info_df`` mapping into a polars frame. + + ``info_df`` is ``{column: {cell_label: value}}`` (pandas "columns" orient). + We parse via pandas (the format is pandas-shaped), guarantee the cell label + is a real ``filename`` column, and hand back polars. + """ + pdf = pd.DataFrame(info_df) + pdf = pdf.dropna(how="all") + if FILENAME not in pdf.columns: + # the cell label only lived in the index -> promote it to a column + pdf = pdf.rename_axis(FILENAME).reset_index() + else: + pdf = pdf.reset_index(drop=True) + return _to_polars(pdf) + + +def _pages_to_info_df(pages: pl.DataFrame) -> dict: + """Serialise pages back to the legacy ``info_df`` mapping.""" + pdf = pages.to_pandas() + if FILENAME in pdf.columns: + pdf = pdf.set_index(FILENAME, drop=False) + return json.loads(pdf.to_json(default_handler=str)) + + +def read_journal(path: Path | str) -> Journal: + """Load a journal from a ``.json`` file into the :class:`Journal` model.""" + path = Path(path) + raw = json.loads(path.read_text(encoding="utf-8")) + if "info_df" not in raw: + raise ValueError(f"not a cellpy journal (missing 'info_df'): {path}") + + meta = raw.get("metadata") or {} + session = raw.get("session") or _empty_session() + for key in keys_journal_session: + session.setdefault(key, None) + + pages = _pages_from_info_df(raw["info_df"]) + return Journal( + name=meta.get("name"), + project=meta.get("project"), + pages=pages, + session=session, + meta=meta, + ) + + +def write_journal(journal: Journal, path: Path | str) -> Path: + """Write a :class:`Journal` to ``path`` in the compatible JSON format.""" + path = Path(path) + meta = dict(journal.meta) + meta.setdefault("name", journal.name) + meta.setdefault("project", journal.project) + top_level = { + "info_df": _pages_to_info_df(journal.pages), + "metadata": meta, + "session": journal.session, + } + path.write_text(json.dumps(top_level, default=str), encoding="utf-8") + return path + + +def journal_from_frame( + frame: pl.DataFrame | pd.DataFrame, + name: str | None = None, + project: str | None = None, +) -> Journal: + """Build a journal from a dataframe of pages (polars or pandas). + + The cell label must be available as a ``filename`` column (or the pandas + index, which is promoted to one). + """ + if isinstance(frame, pl.DataFrame): + pages = frame + else: + pdf = frame + if FILENAME not in pdf.columns: + pdf = pdf.rename_axis(FILENAME).reset_index() + else: + pdf = pdf.reset_index(drop=True) + pages = _to_polars(pdf) + return Journal(name=name, project=project, pages=pages) diff --git a/cellpy/batch/layout.py b/cellpy/batch/layout.py new file mode 100644 index 00000000..4667cc5d --- /dev/null +++ b/cellpy/batch/layout.py @@ -0,0 +1,74 @@ +"""Batch folder layout (batch v3, #698). + +Pure path computation, separated from filesystem side effects. In the legacy +``LabJournal`` code, exporting a batch created directories as a side effect of +``paginate()`` (dumpers.py:22 obtained output folders by *calling* it). Here the +two concerns are split: :class:`BatchPaths` only *computes* paths; the single +:func:`ensure_dirs` function is the only thing that touches the filesystem. + +The layout mirrors the modern ``paginate`` default (batch_journals.py): the +project directory is the current working directory, the batch dump directory is +``/dump`` and raw exports live under ``/raw_data``. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +#: Name of the per-project dump directory (kept identical to the legacy +#: ``DEFAULT_OUTPUT_DIR_NAME`` so existing project folders keep working). +DEFAULT_OUTPUT_DIR_NAME = "dump" + +#: Sub-directory of the dump directory that holds exported raw data. +RAW_SUBDIR = "raw_data" + + +@dataclass(frozen=True) +class BatchPaths: + """Computed, immutable folder layout for one batch. + + Nothing here creates directories -- constructing a ``BatchPaths`` and + reading its properties is free of side effects. Call :func:`ensure_dirs` + to materialise the folders. + """ + + name: str + project: str + project_dir: Path + + @classmethod + def create( + cls, name: str, project: str, project_dir: Path | str | None = None + ) -> "BatchPaths": + """Build a layout; ``project_dir`` defaults to the current directory.""" + base = Path(project_dir) if project_dir is not None else Path.cwd() + return cls(name=name, project=project, project_dir=base) + + @property + def batch_dir(self) -> Path: + """The dump directory for this batch (``/dump``).""" + return self.project_dir / DEFAULT_OUTPUT_DIR_NAME + + @property + def raw_dir(self) -> Path: + """Where exported raw data lives (``/raw_data``).""" + return self.batch_dir / RAW_SUBDIR + + def journal_file(self, suffix: str = ".json") -> Path: + """Path to the journal file for this batch (not created here).""" + return self.project_dir / f"cellpy_batch_{self.name}{suffix}" + + def all_dirs(self) -> tuple[Path, ...]: + """Every directory this layout owns, parents first.""" + return (self.project_dir, self.batch_dir, self.raw_dir) + + +def ensure_dirs(paths: BatchPaths) -> tuple[Path, ...]: + """Create every directory in ``paths`` (idempotent). The *only* mkdir. + + Returns the directories that were ensured (parents first). + """ + for directory in paths.all_dirs(): + directory.mkdir(parents=True, exist_ok=True) + return paths.all_dirs() diff --git a/tests/test_batch_v3.py b/tests/test_batch_v3.py new file mode 100644 index 00000000..5dfc3ec0 --- /dev/null +++ b/tests/test_batch_v3.py @@ -0,0 +1,94 @@ +"""Unit tests for the batch v3 package (#698): journal model + layout.""" + +import polars as pl +import pytest + +from cellpy.batch import ( + BatchPaths, + Journal, + ensure_dirs, + journal_from_frame, + read_journal, + write_journal, +) +from cellpy.batch.journal import FILENAME + + +# ---- journal.py --------------------------------------------------------- + + +def test_read_journal_json(parameters): + j = read_journal(parameters.journal_file_json_path) + assert isinstance(j, Journal) + assert isinstance(j.pages, pl.DataFrame) + assert len(j) == 5 + # keys-in-columns law: the cell label is a column, not an index + assert FILENAME in j.pages.columns + assert "argument" in j.pages.columns + assert len(j.cell_names) == 5 + # session always carries the four canonical keys + assert set(j.session) >= {"starred", "bad_cells", "bad_cycles", "notes"} + + +def test_journal_json_roundtrip(parameters, tmp_path): + j1 = read_journal(parameters.journal_file_json_path) + out = tmp_path / "roundtrip.json" + returned = write_journal(j1, out) + assert returned == out and out.is_file() + + j2 = read_journal(out) + # value parity on the pages after a write -> read cycle + assert j2.cell_names == j1.cell_names + assert set(j2.pages.columns) == set(j1.pages.columns) + order = j1.cell_names + j1s = j1.pages.sort(FILENAME) + j2s = j2.pages.sort(FILENAME) + assert j2s["argument"].to_list() == j1s["argument"].to_list() + assert j2s[FILENAME].to_list() == sorted(order) + + +def test_journal_from_frame_polars(): + pages = pl.DataFrame( + { + FILENAME: ["cell_a", "cell_b"], + "argument": ["recalc=True", "recalc=False"], + "group": [1, 1], + } + ) + j = journal_from_frame(pages, name="t", project="p") + assert j.name == "t" and j.project == "p" + assert j.cell_names == ["cell_a", "cell_b"] + assert len(j) == 2 + + +def test_read_journal_rejects_non_journal(tmp_path): + bad = tmp_path / "notajournal.json" + bad.write_text('{"hello": "world"}', encoding="utf-8") + with pytest.raises(ValueError, match="not a cellpy journal"): + read_journal(bad) + + +# ---- layout.py ---------------------------------------------------------- + + +def test_batchpaths_is_pure(tmp_path): + """Computing paths must not create anything on disk.""" + p = BatchPaths.create("mybatch", "myproject", project_dir=tmp_path) + assert p.batch_dir == tmp_path / "dump" + assert p.raw_dir == tmp_path / "dump" / "raw_data" + assert p.journal_file() == tmp_path / "cellpy_batch_mybatch.json" + # nothing was created just by asking for paths + assert not p.batch_dir.exists() + assert not p.raw_dir.exists() + + +def test_ensure_dirs_is_the_only_mkdir(tmp_path): + p = BatchPaths.create("mybatch", "myproject", project_dir=tmp_path / "proj") + assert not p.project_dir.exists() + made = ensure_dirs(p) + assert p.project_dir.exists() + assert p.batch_dir.exists() + assert p.raw_dir.exists() + assert set(made) == {p.project_dir, p.batch_dir, p.raw_dir} + # idempotent + ensure_dirs(p) From 5470bf159df110c8336fe59274794ac772d57b64 Mon Sep 17 00:00:00 2001 From: jepegit Date: Sun, 26 Jul 2026 21:37:36 +0200 Subject: [PATCH 2/2] feat(batch): custom-JSON reader (#345) + Excel read-only journals (#698) - read_custom_json/journal_from_custom_json: read arbitrary JSON into pages via a source->journal column map; requires a filename mapping (folds #345). - read_journal now dispatches on suffix; .xlsx is read-only (reads the pages sheet + optional meta), and write_journal rejects .xlsx with a clear error (Excel journals are read-only in batch v3). Co-Authored-By: Claude Opus 4.8 --- cellpy/batch/__init__.py | 4 ++ cellpy/batch/journal.py | 88 +++++++++++++++++++++++++++++++++++++++- tests/test_batch_v3.py | 55 +++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 2 deletions(-) diff --git a/cellpy/batch/__init__.py b/cellpy/batch/__init__.py index f7d0ca77..143a1f3a 100644 --- a/cellpy/batch/__init__.py +++ b/cellpy/batch/__init__.py @@ -16,7 +16,9 @@ from cellpy.batch.journal import ( Journal, + journal_from_custom_json, journal_from_frame, + read_custom_json, read_journal, write_journal, ) @@ -27,6 +29,8 @@ "read_journal", "write_journal", "journal_from_frame", + "read_custom_json", + "journal_from_custom_json", "BatchPaths", "ensure_dirs", ] diff --git a/cellpy/batch/journal.py b/cellpy/batch/journal.py index 70f3d775..12dffcaa 100644 --- a/cellpy/batch/journal.py +++ b/cellpy/batch/journal.py @@ -15,6 +15,7 @@ class mixing data model, three file formats, path fixing, selection state and from __future__ import annotations import json +from collections.abc import Mapping from dataclasses import dataclass, field from pathlib import Path @@ -119,8 +120,16 @@ def _pages_to_info_df(pages: pl.DataFrame) -> dict: def read_journal(path: Path | str) -> Journal: - """Load a journal from a ``.json`` file into the :class:`Journal` model.""" + """Load a journal into the :class:`Journal` model. + + ``.json`` is the native, round-trippable format. ``.xlsx`` is supported + **read-only** (a lab convenience); writing Excel is intentionally not + supported in batch v3 (see :func:`write_journal`). + """ path = Path(path) + if path.suffix == ".xlsx": + return _read_journal_excel(path) + raw = json.loads(path.read_text(encoding="utf-8")) if "info_df" not in raw: raise ValueError(f"not a cellpy journal (missing 'info_df'): {path}") @@ -140,9 +149,84 @@ def read_journal(path: Path | str) -> Journal: ) +def _read_journal_excel(path: Path) -> Journal: + """Read the ``pages`` sheet of an Excel journal (read-only).""" + pdf = pd.read_excel(path, sheet_name="pages", engine="openpyxl") + if FILENAME not in pdf.columns: + # legacy Excel writes the cell label as the (index) first column + first = pdf.columns[0] + if str(first).lower().startswith("unnamed") or first == "index": + pdf = pdf.rename(columns={first: FILENAME}) + pdf = pdf.dropna(how="all").reset_index(drop=True) + + meta: dict = {} + try: + mdf = pd.read_excel(path, sheet_name="meta", engine="openpyxl") + if {"parameter", "value"}.issubset(mdf.columns): + meta = dict(zip(mdf["parameter"], mdf["value"])) + except (ValueError, KeyError): + pass + + return Journal( + name=meta.get("name", path.stem), + project=meta.get("project"), + pages=_to_polars(pdf), + meta=meta, + ) + + +def read_custom_json(path: Path | str, column_map: Mapping[str, str]) -> pl.DataFrame: + """Read an arbitrary JSON file into journal pages via a column map (#345). + + ``column_map`` maps *source* JSON keys to cellpy journal keys, e.g. + ``{"cell_id": "filename", "mass_mg": "mass", "instrument_name": "instrument"}``. + The JSON may be a dict of columns (``{key: [values]}``) or a list of + records. At least one source key must map to ``filename``. + """ + data = json.loads(Path(path).read_text(encoding="utf-8")) + pdf = pd.DataFrame(data) + + rename = { + src: _hdr[cellpy_key] + for src, cellpy_key in column_map.items() + if src in pdf.columns + } + pdf = pdf.rename(columns=rename) + keep = [c for c in pdf.columns if c in set(rename.values())] + pdf = pdf[keep] + + if FILENAME not in pdf.columns: + raise ValueError( + "column_map must map a source column to 'filename' " + f"(got mappings to {sorted(set(rename.values()))})" + ) + return _to_polars(pdf.reset_index(drop=True)) + + +def journal_from_custom_json( + path: Path | str, + column_map: Mapping[str, str], + name: str | None = None, + project: str | None = None, +) -> Journal: + """Build a :class:`Journal` from an arbitrary JSON file (#345).""" + return Journal( + name=name, project=project, pages=read_custom_json(path, column_map) + ) + + def write_journal(journal: Journal, path: Path | str) -> Path: - """Write a :class:`Journal` to ``path`` in the compatible JSON format.""" + """Write a :class:`Journal` to ``path`` in the compatible JSON format. + + Only ``.json`` is written. Excel journals are read-only in batch v3 + (metadata plan Step 4); export a report frame instead of a journal. + """ path = Path(path) + if path.suffix == ".xlsx": + raise ValueError( + "Excel journals are read-only in batch v3; write a .json journal " + "instead (use outputs.write_excel for report frames)." + ) meta = dict(journal.meta) meta.setdefault("name", journal.name) meta.setdefault("project", journal.project) diff --git a/tests/test_batch_v3.py b/tests/test_batch_v3.py index 5dfc3ec0..475ff71e 100644 --- a/tests/test_batch_v3.py +++ b/tests/test_batch_v3.py @@ -1,5 +1,7 @@ """Unit tests for the batch v3 package (#698): journal model + layout.""" +import pathlib + import polars as pl import pytest @@ -7,12 +9,16 @@ BatchPaths, Journal, ensure_dirs, + journal_from_custom_json, journal_from_frame, + read_custom_json, read_journal, write_journal, ) from cellpy.batch.journal import FILENAME +FIXTURES = pathlib.Path(__file__).parent / "fixtures" + # ---- journal.py --------------------------------------------------------- @@ -68,6 +74,55 @@ def test_read_journal_rejects_non_journal(tmp_path): read_journal(bad) +# ---- custom JSON (#345) ------------------------------------------------- + + +def test_read_custom_json_with_column_map(): + column_map = { + "cell_id": "filename", + "mass_mg": "mass", + "total_mass_mg": "total_mass", + "instrument_name": "instrument", + } + pages = read_custom_json(FIXTURES / "custom_json_batch_like.json", column_map) + assert isinstance(pages, pl.DataFrame) + assert FILENAME in pages.columns + assert "mass" in pages.columns and "instrument" in pages.columns + assert pages[FILENAME].to_list() == ["20160805_test001_45_cc"] + + j = journal_from_custom_json( + FIXTURES / "custom_json_batch_like.json", + column_map, + name="cj", + project="p", + ) + assert j.name == "cj" + assert j.cell_names == ["20160805_test001_45_cc"] + + +def test_custom_json_requires_filename_mapping(): + with pytest.raises(ValueError, match="must map a source column to 'filename'"): + read_custom_json( + FIXTURES / "custom_json_batch_like.json", {"mass_mg": "mass"} + ) + + +# ---- Excel read-only ---------------------------------------------------- + + +def test_read_journal_excel(parameters): + j = read_journal(parameters.journal_file_full_xlsx_path) + assert isinstance(j.pages, pl.DataFrame) + assert len(j) == 2 + assert FILENAME in j.pages.columns + + +def test_write_journal_rejects_excel(parameters, tmp_path): + j = read_journal(parameters.journal_file_json_path) + with pytest.raises(ValueError, match="read-only"): + write_journal(j, tmp_path / "nope.xlsx") + + # ---- layout.py ----------------------------------------------------------