feat: auto-generate Pydantic models from JSON schema & add CI + linting

.github/workflows/ci.yml

@@ -0,0 +1,81 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  detect-changes:
+    name: Detect changed paths
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    outputs:
+      src: ${{ steps.filter.outputs.src }}
+      models: ${{ steps.filter.outputs.models }}
+    steps:
+      - uses: actions/checkout@v6
+      - uses: dorny/paths-filter@v4
+        id: filter
+        with:
+          filters: |
+            src:
+              - 'src/**'
+              - 'tests/**'
+              - 'pyproject.toml'
+            models:
+              - 'src/gtfs_diff/schema.conf'
+              - 'scripts/generate_models.sh'
+              - 'src/gtfs_diff/models.py'
+              - '.github/workflows/ci.yml'
+
+  test:
+    needs:
+      - detect-changes
+      - lint
+    name: Run tests on Python ${{ matrix.python-version }}
+    if: needs.detect-changes.outputs.src == 'true'
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.12", "3.14"]
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install -e '.[dev]'
+      - run: pytest --tb=short
+
+  lint:
+    needs: detect-changes
+    name: Lint and format check
+    if: needs.detect-changes.outputs.src == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - run: pip install -e '.[dev]'
+      - run: ./scripts/lint.sh
+
+  models-freshness:
+    needs: detect-changes
+    name: Check models.py is up to date
+    if: needs.detect-changes.outputs.models == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - run: pip install 'datamodel-code-generator[ruff]>=0.59'
+      - run: ./scripts/generate_models.sh
+      - name: Check models.py is up to date
+        run: |
+          if ! git diff --exit-code src/gtfs_diff/models.py; then
+            echo "::error::src/gtfs_diff/models.py is out of date with the schema. Run ./scripts/generate_models.sh to regenerate."
+            exit 1
+          fi

pyproject.toml

@@ -17,6 +17,8 @@ dependencies = [
 dev = [
     "pytest>=7.0",
     "pytest-cov",
+    "datamodel-code-generator[ruff]>=0.59",
+    "ruff>=0.11",
 ]
 
 [project.scripts]
@@ -27,3 +29,21 @@ packages = ["src/gtfs_diff"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
+
+[tool.ruff]
+target-version = "py310"
+src = ["src", "tests"]
+exclude = ["src/gtfs_diff/models.py"]
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle errors
+    "F",   # pyflakes
+    "I",   # isort
+    "UP",  # pyupgrade
+    "B",   # flake8-bugbear
+    "SIM", # flake8-simplify
+]
+
+[tool.ruff.format]
+exclude = ["src/gtfs_diff/models.py"]

scripts/generate_models.sh

@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+# Generate Pydantic v2 models from the GTFS Diff JSON Schema.
+#
+# Usage:
+#   ./scripts/generate_models.sh                     # use version from src/gtfs_diff/schema.conf
+#   ./scripts/generate_models.sh v2-rc1              # fetch a specific version from GitHub
+#   ./scripts/generate_models.sh /path/to/local.json # use a local file
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+OUTPUT="$REPO_ROOT/src/gtfs_diff/models.py"
+SCHEMA_REPO="MobilityData/gtfs-diff"
+SCHEMA_BRANCH="main"
+
+# --- Resolve input: argument > schema.conf -----------------------------------
+SCHEMA_CONF="$REPO_ROOT/src/gtfs_diff/schema.conf"
+if [ $# -ge 1 ]; then
+  INPUT="$1"
+elif [ -f "$SCHEMA_CONF" ]; then
+  # shellcheck source=../src/gtfs_diff/schema.conf
+  source "$SCHEMA_CONF"
+  INPUT="${SCHEMA_VERSION:?SCHEMA_VERSION not set in schema.conf}"
+  echo "Using version from schema.conf: $INPUT"
+else
+  echo "Usage: $0 [<version | /path/to/schema.json>]" >&2
+  echo "  e.g. $0 v2-rc1" >&2
+  echo "  or set SCHEMA_VERSION in src/gtfs_diff/schema.conf" >&2
+  exit 1
+fi
+
+# --- Resolve schema source ---------------------------------------------------
+if [ -f "$INPUT" ]; then
+  SCHEMA_FILE="$INPUT"
+  echo "Using local schema: $SCHEMA_FILE"
+else
+  VERSION="$INPUT"
+  SCHEMA_URL="https://raw.githubusercontent.com/$SCHEMA_REPO/$SCHEMA_BRANCH/spec/v2/json_schema/${VERSION}.json"
+  SCHEMA_FILE="$(mktemp "${TMPDIR:-/tmp}/gtfs_diff_schema_XXXXXX.json")"
+  trap 'rm -f "$SCHEMA_FILE"' EXIT
+
+  echo "Fetching schema from $SCHEMA_URL ..."
+  curl -fsSL "$SCHEMA_URL" -o "$SCHEMA_FILE"
+fi
+
+# --- Ensure datamodel-code-generator is available ----------------------------
+if ! command -v datamodel-codegen &>/dev/null; then
+  echo "Installing datamodel-code-generator ..."
+  pip install 'datamodel-code-generator[ruff]>=0.59'
+fi
+
+# --- Generate ----------------------------------------------------------------
+echo "Generating models → $OUTPUT"
+datamodel-codegen \
+  --formatters ruff-format ruff-check \
+  --input "$SCHEMA_FILE" \
+  --input-file-type jsonschema \
+  --output-model-type pydantic_v2.BaseModel \
+  --target-python-version 3.10 \
+  --use-standard-collections \
+  --use-union-operator \
+  --snake-case-field \
+  --collapse-root-models \
+  --enum-field-as-literal all \
+  --field-constraints \
+  --use-schema-description \
+  --extra-fields allow \
+  --class-name GtfsDiff \
+  --output "$OUTPUT"
+
+# --- Post-process: replace header, append __all__ ----------------------------
+# Replace the codegen header with a clear auto-generated notice.
+{
+  echo "# AUTO-GENERATED — DO NOT EDIT"
+  echo "# This file is generated from the GTFS Diff JSON Schema."
+  echo "# To regenerate: ./scripts/generate_models.sh"
+  echo "# Schema source: https://github.com/$SCHEMA_REPO"
+  echo ""
+  # Strip the original codegen comment block (everything before the first blank line).
+  sed -n '/^$/,$p' "$OUTPUT"
+} > "$OUTPUT.tmp" && mv "$OUTPUT.tmp" "$OUTPUT"
+
+# Collect class names and append __all__.
+CLASSES=$(grep -oE '^class ([A-Za-z_][A-Za-z0-9_]*)' "$OUTPUT" | awk '{print $2}')
+{
+  echo ""
+  echo ""
+  echo "__all__ = ["
+  for cls in $CLASSES; do
+    echo "    \"$cls\","
+  done
+  echo "]"
+} >> "$OUTPUT"
+
+COUNT=$(echo "$CLASSES" | wc -w | tr -d ' ')
+echo "Done — $COUNT model(s) generated."

scripts/lint-fix.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Auto-fix lint violations and reformat code.
+# Usage: ./scripts/lint-fix.sh
+set -euo pipefail
+
+echo "Fixing lint violations ..."
+ruff check --fix src/ tests/
+
+echo "Formatting ..."
+ruff format src/ tests/
+
+echo "Done."

scripts/lint.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Check lint and formatting (exits non-zero on violations).
+# Usage: ./scripts/lint.sh
+set -euo pipefail
+
+echo "Checking lint rules ..."
+ruff check src/ tests/
+
+echo "Checking formatting ..."
+ruff format --check src/ tests/
+
+echo "All clean."

src/gtfs_diff/cli.py

@@ -13,16 +13,33 @@
 @click.version_option(version="0.1.0", prog_name="gtfs-diff-engine")
 @click.argument("base_feed", type=click.Path(exists=True, path_type=Path))
 @click.argument("new_feed", type=click.Path(exists=True, path_type=Path))
-@click.option("--output", "-o", type=click.Path(path_type=Path), default=None,
-              help="Write JSON output to FILE instead of stdout.")
-@click.option("--cap", "-c", type=int, default=None,
-              help="Max row changes per file (0 = omit row-level detail).")
-@click.option("--pretty/--no-pretty", default=True,
-              help="Pretty-print JSON (default: --pretty).")
-@click.option("--base-downloaded-at", default=None,
-              help="ISO 8601 datetime for when base was downloaded.")
-@click.option("--new-downloaded-at", default=None,
-              help="ISO 8601 datetime for when new was downloaded.")
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(path_type=Path),
+    default=None,
+    help="Write JSON output to FILE instead of stdout.",
+)
+@click.option(
+    "--cap",
+    "-c",
+    type=int,
+    default=None,
+    help="Max row changes per file (0 = omit row-level detail).",
+)
+@click.option(
+    "--pretty/--no-pretty", default=True, help="Pretty-print JSON (default: --pretty)."
+)
+@click.option(
+    "--base-downloaded-at",
+    default=None,
+    help="ISO 8601 datetime for when base was downloaded.",
+)
+@click.option(
+    "--new-downloaded-at",
+    default=None,
+    help="ISO 8601 datetime for when new was downloaded.",
+)
 def main(
     base_feed: Path,
     new_feed: Path,
@@ -38,8 +55,12 @@ def main(
     NEW_FEED:  path to the new GTFS feed (zip or directory)
     """
     try:
-        base_dt = datetime.fromisoformat(base_downloaded_at) if base_downloaded_at else None
-        new_dt = datetime.fromisoformat(new_downloaded_at) if new_downloaded_at else None
+        base_dt = (
+            datetime.fromisoformat(base_downloaded_at) if base_downloaded_at else None
+        )
+        new_dt = (
+            datetime.fromisoformat(new_downloaded_at) if new_downloaded_at else None
+        )
     except ValueError as exc:
         click.echo(f"Error: {exc}", err=True)
         sys.exit(1)
@@ -55,7 +76,8 @@ def main(
     except MissingPrimaryKeyError as exc:
         click.echo(
             f"ERROR: Cannot process '{exc.file_name}' — "
-            f"required primary key column(s) {exc.missing_columns} are missing from the file headers.\n"
+            f"required primary key column(s) {exc.missing_columns} "
+            f"are missing from the file headers.\n"
             f"Headers found: {exc.headers}",
             err=True,
         )
@@ -64,7 +86,10 @@ def main(
         click.echo(f"Error: {exc}", err=True)
         sys.exit(1)
 
-    json_str = result.model_dump_json(indent=2, exclude_none=True) if pretty else result.model_dump_json(exclude_none=True)
+    if pretty:
+        json_str = result.model_dump_json(indent=2, exclude_none=True)
+    else:
+        json_str = result.model_dump_json(exclude_none=True)
 
     if output is not None:
         try: