V0.6.19

Makefile.dev

@@ -58,7 +58,23 @@ act-commit:
 	--env VIRTUAL_ENV=
 
 concat-docs:
-	$(UV) run python _scripts/concat_docs.py -o Combined.md
+	$(UV) run python _scripts/concat_docs.py -o _exports/Combined_Docs.md --exclude "examples/**"
 
 export-demo:
-	$(UV) python _scripts/export_subdir_md.py examples/incremental_demo -o _exports/incremental_demo_export.md --exclude-ext html css
+	$(UV) python _scripts/export_subdir_md.py examples/incremental_demo -o _exports/incremental_demo_export.md --exclude-ext html css
+
+show-structure:
+	tree -a -I '.git|.venv|__pycache__|.pytest_cache|.mypy_cache|.ruff_cache|node_modules|dist|build|htmlcov|site|.fastflowtransform|.DS_Store|.idea|.vscode|.local|metastore_db|.uv-cache|_exports|_scripts|articles|examples_article|tickets'
+
+spinup-pg:
+	docker rm -f ff_pg
+	docker volume rm pgdata
+	docker volume create pgdata
+
+	docker run -d --name ff_pg \
+	-e POSTGRES_USER=postgres \
+	-e POSTGRES_PASSWORD=postgres \
+	-e POSTGRES_DB=ffdb \
+	-p 5432:5432 \
+	-v pgdata:/var/lib/postgresql/data \
+	postgres:17

docs/Auto_Docs.md

@@ -30,6 +30,62 @@ fft docgen . --env dev --out site/docs --emit-json site/docs/docs_manifest.json
 
 This generates the SPA and also writes a manifest you can use for CI checks or custom tooling.
 
+Note: If `artifacts.mode` is set to `db`, docs generation is disabled because it would create local asset files (SPA JS/CSS and JSON). Switch to `files` or `both` to generate docs.
+
+Artifacts configuration lives in your `profiles.yml` under the selected environment. Example:
+
+```yaml
+dev_postgres:
+  engine: postgres
+  postgres:
+    dsn: "{{ env('FF_PG_DSN') }}"
+    db_schema: "{{ env('FF_PG_SCHEMA', 'public') }}"
+
+  artifacts:
+    mode: both          # files | db | both
+    engine: postgres    # currently only postgres supported
+    postgres:
+      dsn: "{{ env('FF_ARTIFACTS_PG_DSN') }}"
+      db_schema: "{{ env('FF_ARTIFACTS_PG_SCHEMA', 'public') }}"  # or `schema: ...`
+```
+
+Env overrides:
+- `FF_ARTIFACTS_MODE` overrides `artifacts.mode`.
+- `FF_ARTIFACTS_PG_DSN` overrides `artifacts.postgres.dsn`.
+- `FF_ARTIFACTS_PG_SCHEMA` overrides `artifacts.postgres.db_schema` (or `schema`).
+
+Validation rules:
+- `mode: files` does not require any Postgres settings.
+- `mode: db` or `both` requires `artifacts.engine=postgres` and a valid `dsn` + `db_schema`.
+
+Frontend contract (SPA data sources):
+- The SPA reads globals (if present) to resolve artifact URLs:
+  - `__FFT_MANIFEST_PATH__`
+  - `__FFT_RUN_RESULTS_PATH__`
+  - `__FFT_TEST_RESULTS_PATH__`
+  - `__FFT_UTEST_RESULTS_PATH__`
+  - `__FFT_ARTIFACTS_API_BASE__`
+  - `__FFT_ENV__`
+  - `__FFT_ENGINE__`
+  - `__FFT_RUN_ID__`
+- URL resolution order:
+  1. Explicit `__FFT_*_PATH__` globals
+  2. API base + params (`__FFT_ARTIFACTS_API_BASE__` with env/engine/run_id)
+  3. Query params (`?manifest=...&run=...&test=...&utest=...`)
+  4. Local assets (`assets/*.json`)
+- Expected endpoint shapes (examples):
+  - Latest by env/engine:
+    - `/artifacts/latest/docs_manifest?env=<env>&engine=<engine>`
+    - `/artifacts/latest/run_results?env=<env>&engine=<engine>`
+    - `/artifacts/latest/test_results?env=<env>&engine=<engine>`
+    - `/artifacts/latest/utest_results?env=<env>&engine=<engine>`
+  - Specific run:
+    - `/artifacts/run/<run_id>/docs_manifest`
+    - `/artifacts/run/<run_id>/run_results`
+    - `/artifacts/run/<run_id>/test_results`
+    - `/artifacts/run/<run_id>/utest_results`
+- Response payload can be either raw JSON or wrapped as `{ payload: {...} }` or `{ data: {...} }`.
+
 ### Classic (DAG-only)
 
 ```bash
@@ -353,6 +409,7 @@ If your build outputs runtime artifacts (run results, tests), the portal can dis
 - optional unit test results
 
 This is intended to be lightweight and opt-in: docs remain usable without it.
+In `artifacts.mode=db`, runtime artifacts are stored in Postgres and the docs site is not generated, so the health strip is unavailable.
 
 ---
 

examples/basic_demo/.env.dev_postgres

@@ -1,3 +1,7 @@
 # Postgres profile for the basic demo (replace with your own connection string)
 FF_PG_DSN=postgresql+psycopg://postgres:postgres@localhost:5432
 FF_PG_SCHEMA=basic_demo
+
+FF_ARTIFACTS_MODE=both
+FF_ARTIFACTS_PG_DSN=postgresql+psycopg://postgres:postgres@localhost:5432
+FF_ARTIFACTS_PG_SCHEMA=basic_demo

examples/basic_demo/profiles.yml

@@ -18,6 +18,13 @@ dev_postgres:
     dsn: "{{ env('FF_PG_DSN') }}"
     db_schema: "{{ env('FF_PG_SCHEMA', 'public') }}"
 
+  artifacts:
+    mode: both          # files | db | both   (default: files)
+    engine: postgres    # currently only postgres supported
+    postgres:
+      dsn: "{{ env('FF_PG_DSN') }}"
+      db_schema: "{{ env('FF_PG_SCHEMA', 'public') }}"
+
 dev_postgres_utest:
   engine: postgres        # same engine
   postgres:

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "fastflowtransform"
-version = "0.6.18"
+version = "0.6.19"
 description = "Python framework for SQL & Python data transformation, ETL pipelines, and dbt-style data modeling"
 readme = "README.md"
 license = { text = "Apache-2.0" }

src/fastflowtransform/artifacts/__init__.py

@@ -0,0 +1,27 @@
+"""Artifacts helpers (file-based + builders)."""
+
+from __future__ import annotations
+
+from .files import (
+    RunNodeResult,
+    TestResult,
+    UTestResult,
+    load_last_run_durations,
+    write_catalog,
+    write_manifest,
+    write_run_results,
+    write_test_results,
+    write_utest_results,
+)
+
+__all__ = [
+    "RunNodeResult",
+    "TestResult",
+    "UTestResult",
+    "load_last_run_durations",
+    "write_catalog",
+    "write_manifest",
+    "write_run_results",
+    "write_test_results",
+    "write_utest_results",
+]

src/fastflowtransform/artifacts/config.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from fastflowtransform.logging import warn
+from fastflowtransform.settings import _render_profiles_template
+
+
+@dataclass(frozen=True)
+class ResolvedArtifactsDb:
+    mode: str  # files|db|both
+    dsn: str
+    db_schema: str
+
+
+def _read_profiles_yaml(project_dir: Path) -> dict[str, Any]:
+    for name in ("profiles.yml", "profiles.yaml"):
+        p = project_dir / name
+        if p.exists():
+            raw_text = p.read_text(encoding="utf-8")
+            rendered = _render_profiles_template(raw_text, project_dir)
+            raw = yaml.safe_load(rendered) or {}
+            return raw if isinstance(raw, dict) else {}
+    return {}
+
+
+def resolve_artifacts_db(project_dir: Path, env_name: str) -> ResolvedArtifactsDb | None:
+    """
+    Reads project_dir/profiles.yml and looks for:
+
+      <env_name>.artifacts.mode
+      <env_name>.artifacts.postgres.dsn
+      <env_name>.artifacts.postgres.db_schema
+
+    Also supports an optional top-level 'profiles:' wrapper:
+
+      profiles:
+        dev: {...}
+    """
+    raw = _read_profiles_yaml(project_dir)
+    if not raw:
+        return None
+
+    profiles = raw.get("profiles")
+    root = profiles if isinstance(profiles, dict) else raw
+
+    env = root.get(env_name)
+    if not isinstance(env, dict):
+        return None
+
+    artifacts = env.get("artifacts")
+    if not isinstance(artifacts, dict):
+        return None
+
+    mode = str(artifacts.get("mode") or "files").lower().strip()
+    if mode not in ("files", "db", "both"):
+        warn(f"[artifacts] invalid artifacts.mode={mode!r}; falling back to 'files'")
+        mode = "files"
+
+    engine = str(artifacts.get("engine") or "postgres").lower().strip()
+    if engine != "postgres":
+        warn(f"[artifacts] artifacts.engine={engine!r} not supported yet; ignoring")
+        return None
+
+    pg = artifacts.get("postgres")
+    if not isinstance(pg, dict):
+        return None
+
+    dsn = pg.get("dsn")
+    schema = pg.get("db_schema") or pg.get("schema")
+    if not isinstance(dsn, str) or not dsn.strip():
+        return None
+    if not isinstance(schema, str) or not schema.strip():
+        return None
+
+    return ResolvedArtifactsDb(mode=mode, dsn=dsn.strip(), db_schema=schema.strip())