Tutorial Notebook Bug Fix + Notebook CI Flow + Merging 5T-OTA Example #87, 92

.github/workflows/notebooks.yml

@@ -0,0 +1,147 @@
+name: Tutorial Notebooks
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+# Cancel superseded runs on the same branch — pushing N commits in a row
+# shouldn't keep N CI runs going.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  notebooks:
+    name: Run tutorial notebooks
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+
+    # Same container + Python setup as drc.yml, so the two workflows can
+    # share a venv cache. iic-osic-tools ships klayout, magic, netgen, both
+    # PDKs (under /foss/pdks), but Python 3.12 + gdsfactory 9.x — we install
+    # CPython 3.10 via uv and let glayout's pinned gdsfactory 7.7 / numpy 1.x
+    # come along.
+    container:
+      image: hpretl/iic-osic-tools:latest
+      options: --user root
+      env:
+        # Both sky130A and gf180mcuD live here; glayout's gf180 module reads
+        # this at import time even when a notebook only touches sky130, so
+        # it has to be set globally.
+        PDK_ROOT: /foss/pdks
+        # FVF / INV / BJT tutorials default to gf180; the bootstrap cell in
+        # each notebook derives PDKPATH from PDK_ROOT/PDK if unset.
+        PDK: gf180mcuD
+        PDKPATH: /foss/pdks/gf180mcuD
+        DEBIAN_FRONTEND: noninteractive
+        PYTHONUNBUFFERED: "1"
+        # The image sets PYTHONPATH to its 3.12 site-packages, which would
+        # leak gdsfactory 9.x into the 3.10 kernel.
+        PYTHONPATH: ""
+        # GitHub Actions overrides the image's ENTRYPOINT with `tail -f`,
+        # so the iic-osic-tools entrypoint that normally enriches PATH with
+        # /foss/tools/{bin,klayout,...} never runs.
+        PATH: /foss/tools/bin:/foss/tools/sak:/foss/tools/kactus2:/foss/tools/klayout:/foss/tools/osic-multitool:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+    # iic-osic-tools' default container shell is dash, which doesn't grok
+    # the bash features uv's installer and our notebooks rely on.
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check that notebook outputs are cleared
+        # Fail fast (sub-second, stdlib-only) before the multi-minute
+        # nbconvert pass. Catches PRs that ship notebooks with stale outputs
+        # — those bloat the diff and defeat the cleared-outputs convention
+        # the repo settled on in PR #89.
+        run: python3 tests/notebooks/check_outputs_cleared.py
+
+      - name: Cache uv + CPython 3.10
+        id: cache-uv
+        uses: actions/cache@v4
+        with:
+          # HOME=/headless in the image even when running as root.
+          path: |
+            /headless/.local/bin/uv
+            /headless/.local/bin/uvx
+            /headless/.local/share/uv
+          key: uv-py310-${{ runner.os }}-v1
+
+      - name: Install Python 3.10 (uv)
+        run: |
+          set -euxo pipefail
+          if [ ! -x "$HOME/.local/bin/uv" ]; then
+            curl -LsSf https://astral.sh/uv/install.sh | sh
+          fi
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+          export PATH="$HOME/.local/bin:$PATH"
+          uv python install 3.10
+          echo "PYTHON310=$(uv python find 3.10)" >> "$GITHUB_ENV"
+
+      - name: Show tool versions
+        run: |
+          set -euxo pipefail
+          klayout -v
+          magic -d null -noconsole -T minimum </dev/null 2>&1 | head -5 || true
+          netgen -batch lvs -version 2>&1 | head -3 || true
+          ngspice -v 2>&1 | head -3 || true
+          "$PYTHON310" --version
+          ls "$PDK_ROOT"
+
+      - name: Cache python venv
+        id: cache-venv
+        uses: actions/cache@v4
+        with:
+          path: ${{ github.workspace }}/.venv
+          # Bust the cache on dependency-relevant changes. notebooks adds
+          # jupyter/nbconvert/ipykernel/ipywidgets/gdstk/svgutils on top of
+          # the DRC venv, so a separate cache key avoids touching the
+          # drc-venv cache.
+          key: notebooks-venv-py310-${{ runner.os }}-${{ hashFiles('setup.py', 'src/glayout/**/*.py') }}-v1
+          restore-keys: |
+            notebooks-venv-py310-${{ runner.os }}-
+
+      - name: Create venv and install glayout + jupyter (cache miss)
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: |
+          set -euxo pipefail
+          rm -rf "$GITHUB_WORKSPACE/.venv"
+          "$PYTHON310" -m venv "$GITHUB_WORKSPACE/.venv"
+          . "$GITHUB_WORKSPACE/.venv/bin/activate"
+          # uv pip install is ~3-5x faster than pip and auto-detects
+          # $VIRTUAL_ENV after `activate`. glayout's install_requires already
+          # pulls in gdstk / svgutils / ipywidgets; jupyter+nbconvert+
+          # ipykernel are the only extras the notebook harness needs.
+          uv pip install -e .
+          uv pip install jupyter nbconvert ipykernel
+
+      - name: Run all tutorial notebooks
+        run: |
+          set -euxo pipefail
+          . "$GITHUB_WORKSPACE/.venv/bin/activate"
+          python tests/notebooks/run_tutorial_notebooks.py \
+            --out-dir notebook_results
+
+      - name: Upload notebook artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: tutorial-notebooks
+          # Includes executed/*.ipynb, logs/*.log, summary.json, junit.xml —
+          # everything needed to triage a regression without rerunning CI.
+          path: notebook_results
+          retention-days: 14
+
+      - name: Publish JUnit summary
+        # Skip when junit.xml wasn't produced (the runner died before write).
+        if: ${{ always() && hashFiles('notebook_results/junit.xml') != '' }}
+        uses: mikepenz/action-junit-report@v4
+        with:
+          report_paths: notebook_results/junit.xml
+          check_name: Tutorial notebooks
+          require_tests: true

.gitignore

@@ -256,6 +256,29 @@ cython_debug/
 *.sim
 .*.*
 .*/
+# Re-include project directories that start with a dot but aren't user state
+# (workflow defs, etc.). Without these negations the `.*/` rule above hides
+# anything new under .github/ from `git status`.
+!.github/
+!.github/**
 _*.json
 out/
-drc_results/
+drc_results/
+notebook_results/
+
+# tutorial-time generated artifacts: helper modules written by part 1 of
+# FVF/INV, scratch GDS/SVGs used for inline display, klayout DRC reports,
+# magic extraction directories. None of these should be committed.
+tutorial/FVF/
+tutorial/INV/
+tutorial/ext/
+tutorial/klayout_drc/
+tutorial/out/
+tutorial/BJT_tutorials/ext/
+tutorial/BJT_tutorials/out/
+tutorial/*.gds
+tutorial/*.svg
+tutorial/*.lyrdb
+tutorial/fvf.gds
+tutorial/out.gds
+tutorial/out.svg

src/glayout/pdk/gf180_mapped/gf180_mapped.py

@@ -105,7 +105,7 @@
 
 # note for DRC, there is mim_option 'A'. This is the one configured for use
 
-gf180_lydrc_file_path = Path(__file__).resolve().parent / "gf180mcu_drc.lydrc"
+gf180_lydrc_file_path = Path(__file__).resolve().parent / "gf180mcu_drc_wrapper.drc"
 # openfasoc_dir = Path(__file__).resolve().parent.parent.parent.parent.parent.parent.parent
 # pdk_root = Path('/usr/bin/miniconda3/share/pdk/')
 pdk_root = Path(os.getenv('PDK_ROOT'))

src/glayout/pdk/gf180_mapped/gf180mcu_drc_wrapper.drc

@@ -0,0 +1,14 @@
+# Bridge MappedPDK.drc()'s '-rd in_gds=... -rd report_file=...' (klayout > 0.29
+# convention) into the variables the standalone gf180mcu.drc deck reads
+# ($input / $report). Also presets variant-A metal stack so the deck doesn't
+# bail out. We use eval(File.read(...)) instead of Ruby's load() because the
+# bundled deck uses klayout DRC DSL methods (source, polygons, report, ...)
+# that only exist in the enclosing DRC interpreter scope.
+$input  = $in_gds      if defined?($in_gds)      && $in_gds      && $input.nil?
+$report = $report_file if defined?($report_file) && $report_file && $report.nil?
+$run_mode   ||= "flat"
+$metal_top  ||= "30K"
+$metal_level||= "3LM"
+$mim_option ||= "A"
+$thr        ||= 4
+eval(File.read(File.join(File.dirname(File.expand_path(__FILE__)), "gf180mcu.drc")))

src/glayout/pdk/mappedpdk.py

@@ -874,14 +874,19 @@ def write_spice(input_cdl, output_spice, lvs_schematic_ref_file):
                         print(content)
                         print("==== SPICE MAG END ====")
 
-                lvssetup_file = self.pdk_files['lvs_setup_tcl_file'] if lvs_setup_tcl_file is None else lvs_setup_tcl_file 
-                netgen_command = f'netgen -batch lvs "{str(lvsmag_path)} {design_name}" "{str(spice_path)} {design_name}" {lvssetup_file} {str(report_path)}'
+                lvssetup_file = self.pdk_files['lvs_setup_tcl_file'] if lvs_setup_tcl_file is None else lvs_setup_tcl_file
+                # The netgen wrapper ships with `#!/bin/sh` but uses bashisms
+                # (e.g. `${i//\"/\\\"}`) that explode under dash. Force bash by
+                # passing the script to it directly.
+                _netgen_bin = shutil.which('netgen') or 'netgen'
+                netgen_command = f'bash {_netgen_bin} -batch lvs "{str(lvsmag_path)} {design_name}" "{str(spice_path)} {design_name}" {lvssetup_file} {str(report_path)}'
                 print(f"Running netgen command: {netgen_command.strip()}")
                 netgen_subproc = subprocess.run(
                     netgen_command,
                     shell=True,
-                    check=True, 
-                    capture_output=True
+                    check=True,
+                    capture_output=True,
+                    executable='/bin/bash',
                 )
                 netgen_subproc_code = netgen_subproc.returncode
                 netgen_subproc_out = netgen_subproc.stdout.decode('utf-8')

tests/notebooks/check_outputs_cleared.py

@@ -0,0 +1,119 @@
+"""Fail if any tutorial notebook ships with cell outputs or execution counts.
+
+Run as a CI precheck before ``run_tutorial_notebooks.py``. Stdlib-only so it
+can run before any venv is set up — fails in a fraction of a second when a
+contributor forgets to strip outputs, instead of after the multi-minute
+nbconvert pass.
+
+Usage:
+    python tests/notebooks/check_outputs_cleared.py
+    python tests/notebooks/check_outputs_cleared.py --fix     # strip in-place
+
+A code cell counts as dirty if either:
+  * ``execution_count`` is not None (i.e. the cell has been run), or
+  * ``outputs`` is non-empty.
+
+Markdown / raw cells are ignored. Cell metadata (e.g. ``metadata.execution``,
+``metadata.widgets``) is left alone — we only care about renderable outputs.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import List, Tuple
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+TUTORIAL_DIR = REPO_ROOT / "tutorial"
+
+
+def _scan(nb_path: Path) -> List[Tuple[int, str]]:
+    """Return [(cell_index, reason), ...] for every dirty code cell."""
+    try:
+        nb = json.loads(nb_path.read_text())
+    except (OSError, json.JSONDecodeError) as e:
+        return [(-1, f"unreadable notebook: {e}")]
+    dirty: List[Tuple[int, str]] = []
+    for i, c in enumerate(nb.get("cells", [])):
+        if c.get("cell_type") != "code":
+            continue
+        if c.get("execution_count") is not None:
+            dirty.append((i, f"execution_count={c['execution_count']!r}"))
+        if c.get("outputs"):
+            dirty.append((i, f"{len(c['outputs'])} output(s)"))
+    return dirty
+
+
+def _clear(nb_path: Path) -> bool:
+    """Strip outputs in-place. Returns True if the file changed."""
+    nb = json.loads(nb_path.read_text())
+    changed = False
+    for c in nb.get("cells", []):
+        if c.get("cell_type") != "code":
+            continue
+        if c.get("execution_count") is not None:
+            c["execution_count"] = None
+            changed = True
+        if c.get("outputs"):
+            c["outputs"] = []
+            changed = True
+    if changed:
+        nb_path.write_text(json.dumps(nb, indent=1, ensure_ascii=False) + "\n")
+    return changed
+
+
+def main() -> int:
+    p = argparse.ArgumentParser()
+    p.add_argument("--fix", action="store_true",
+                   help="Strip outputs in-place instead of just reporting.")
+    args = p.parse_args()
+
+    nbs = sorted(
+        nb for nb in TUTORIAL_DIR.rglob("*.ipynb")
+        if ".ipynb_checkpoints" not in nb.parts
+    )
+    if not nbs:
+        print("no tutorial notebooks found", file=sys.stderr)
+        return 2
+
+    bad: List[Tuple[Path, List[Tuple[int, str]]]] = []
+    for nb in nbs:
+        if args.fix:
+            if _clear(nb):
+                print(f"[FIXED] {nb.relative_to(REPO_ROOT)}")
+            continue
+        dirty = _scan(nb)
+        if dirty:
+            bad.append((nb, dirty))
+
+    if args.fix:
+        return 0
+
+    if not bad:
+        print(f"OK: all {len(nbs)} tutorial notebook(s) have cleared outputs.")
+        return 0
+
+    print("FAIL: the following notebooks ship with non-cleared outputs:\n", file=sys.stderr)
+    for nb, dirty in bad:
+        rel = nb.relative_to(REPO_ROOT)
+        # Cap the noise — one line per offending cell, first 10 cells per nb.
+        shown = dirty[:10]
+        more = len(dirty) - len(shown)
+        for i, why in shown:
+            print(f"  {rel}: cell {i}: {why}", file=sys.stderr)
+        if more > 0:
+            print(f"  {rel}: ... and {more} more", file=sys.stderr)
+    print(
+        "\nFix:\n"
+        "  jupyter nbconvert --clear-output --inplace tutorial/**/*.ipynb tutorial/**/**/*.ipynb\n"
+        "or:\n"
+        "  python tests/notebooks/check_outputs_cleared.py --fix",
+        file=sys.stderr,
+    )
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())