From 7b753d4bc3723d922ee6e13b2a2e1d28ebdee99f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Tue, 9 Jun 2026 15:03:31 +0200 Subject: [PATCH 01/12] test(sync): add CLI behavior tests for project-selection guards and dry-run safety Pin the behaviors that make sync safe and that distinguish kbagent's orchestrator model from kbc's cwd-per-folder model: sync pull/diff/push require --project or --all-projects (and the two are mutually exclusive), --branch is per-project, and push --dry-run propagates the dry-run flag without writing. 6 tests via CliRunner with a mocked SyncService. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_sync_cli_behavior.py | 145 ++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 tests/test_sync_cli_behavior.py diff --git a/tests/test_sync_cli_behavior.py b/tests/test_sync_cli_behavior.py new file mode 100644 index 00000000..2beed17d --- /dev/null +++ b/tests/test_sync_cli_behavior.py @@ -0,0 +1,145 @@ +"""Behavioral CLI tests for the sync command surface. + +These tests pin the *command behaviors* that make sync safe and that distinguish +the kbagent orchestrator model from kbc's cwd-per-folder model: + + 1. ``sync pull/diff/push`` REQUIRE ``--project ALIAS`` or ``--all-projects`` + (kbagent resolves projects from a central config store, it does not act on + the current directory implicitly). Missing/contradictory selection is a + usage error (exit 2) and must NOT reach the service. + 2. ``--project`` and ``--all-projects`` are mutually exclusive. + 3. ``--branch`` is per-project, so it cannot combine with ``--all-projects``. + 4. ``sync push --dry-run`` must call the service in dry-run mode and never + perform a real write. + +Background: a side-by-side comparison against kbc showed that pulls round-trip to +zero drift and that push is last-write-wins; these guards are the first line of +defense against an accidental wrong-target or whole-tree operation. +""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.project_service import ProjectService + +TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" +runner = CliRunner() + + +def _store(config_dir: Path) -> ConfigStore: + config_dir.mkdir(parents=True, exist_ok=True) + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="prod", + project_id=1234, + ), + ) + return store + + +def _invoke(args: list[str], tmp_path: Path) -> tuple[int, MagicMock]: + """Invoke the CLI with a mocked SyncService; return (exit_code, mock).""" + store = _store(tmp_path / "config") + mock_sync = MagicMock() + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProj, + patch("keboola_agent_cli.cli.SyncService") as MockSync, + ): + MockStore.return_value = store + MockProj.return_value = ProjectService(config_store=store) + MockSync.return_value = mock_sync + result = runner.invoke(app, args) + return result.exit_code, mock_sync + + +class TestProjectSelectionRequired: + """pull/diff/push must be told WHICH project(s) — no implicit cwd target.""" + + def test_pull_without_project_is_usage_error(self, tmp_path: Path) -> None: + code, mock = _invoke(["sync", "pull", "--directory", str(tmp_path)], tmp_path) + assert code == 2 + mock.pull.assert_not_called() + mock.pull_all.assert_not_called() + + def test_diff_without_project_is_usage_error(self, tmp_path: Path) -> None: + code, mock = _invoke(["sync", "diff", "--directory", str(tmp_path)], tmp_path) + assert code == 2 + mock.diff.assert_not_called() + + def test_push_without_project_is_usage_error(self, tmp_path: Path) -> None: + code, mock = _invoke(["sync", "push", "--directory", str(tmp_path)], tmp_path) + assert code == 2 + mock.push.assert_not_called() + + +class TestMutuallyExclusiveSelection: + """--project and --all-projects cannot be combined; --branch is per-project.""" + + def test_pull_project_and_all_projects_conflict(self, tmp_path: Path) -> None: + code, mock = _invoke( + ["sync", "pull", "--project", "prod", "--all-projects", "--directory", str(tmp_path)], + tmp_path, + ) + assert code == 2 + mock.pull.assert_not_called() + mock.pull_all.assert_not_called() + + def test_pull_branch_with_all_projects_conflict(self, tmp_path: Path) -> None: + code, mock = _invoke( + ["sync", "pull", "--all-projects", "--branch", "555", "--directory", str(tmp_path)], + tmp_path, + ) + assert code == 2 + mock.pull_all.assert_not_called() + + +class TestPushDryRunIsSafe: + """push --dry-run must run in dry-run mode and never write.""" + + def test_push_dry_run_passes_flag_and_does_not_error(self, tmp_path: Path) -> None: + store = _store(tmp_path / "config") + mock_sync = MagicMock() + # JSON mode avoids the human formatter's field expectations; the point of + # this test is that --dry-run reaches the service, not the print layout. + mock_sync.push.return_value = { + "status": "dry_run", + "project_alias": "prod", + "summary": {"to_create": 0, "to_update": 1, "to_delete": 0}, + "changes": [], + } + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProj, + patch("keboola_agent_cli.cli.SyncService") as MockSync, + ): + MockStore.return_value = store + MockProj.return_value = ProjectService(config_store=store) + MockSync.return_value = mock_sync + result = runner.invoke( + app, + [ + "--json", + "sync", + "push", + "--project", + "prod", + "--dry-run", + "--directory", + str(tmp_path), + ], + ) + assert result.exit_code == 0, result.output + assert mock_sync.push.call_count == 1 + # The dry_run flag must be propagated to the service (no real write). + _, kwargs = mock_sync.push.call_args + assert kwargs.get("dry_run") is True From ff64420bc2b64e35aec57fef22bfe0b1b84dbb28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Tue, 9 Jun 2026 15:03:32 +0200 Subject: [PATCH 02/12] feat(skill): add kbc->kbagent CI/CD migration skill Adds the kbagent-cicd-migration skill: a guided, evidence-based runbook plus a stdlib-only generator (scripts/migrate_cicd.py) that discovers projects from .keboola/manifest.json, detects the legacy kbc CI it replaces, and emits clean kbagent-native validate/pull/push GitHub workflows using uv tool install + kbagent sync with the KBAGENT_PROJECT_FROM_ENV auth model. References cover the kbc<->kbagent command/flag/env mapping, GitHub secrets/environments setup, the one-time breaking-conversion runbook (verified against a live project: the adopt-existing 136-delete footgun and the orphaned config.json cleanup), and the single-branch vs git-branching decision guide. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../skills/kbagent-cicd-migration/SKILL.md | 173 ++++++++ .../references/branching-model.md | 67 +++ .../references/command-mapping.md | 59 +++ .../references/migration-runbook.md | 93 ++++ .../references/secrets-setup.md | 52 +++ .../scripts/migrate_cicd.py | 406 ++++++++++++++++++ 6 files changed, 850 insertions(+) create mode 100644 plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md create mode 100644 plugins/kbagent/skills/kbagent-cicd-migration/references/branching-model.md create mode 100644 plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md create mode 100644 plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md create mode 100644 plugins/kbagent/skills/kbagent-cicd-migration/references/secrets-setup.md create mode 100755 plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md new file mode 100644 index 00000000..78b2240b --- /dev/null +++ b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md @@ -0,0 +1,173 @@ +--- +name: kbagent-cicd-migration +description: > + Use when migrating an existing kbc (keboola-as-code) GitHub CI/CD pipeline to + the new kbagent (keboola-agent-cli) sync engine. Covers: converting per-project + pull/push PR workflows, multi-project repos (e.g. L0/L1 dev->prod promotion), + branch->environment mapping, GitHub secrets/variables/environments setup, the + install step (uv tool install instead of downloading a Go binary), and the + kbc->kbagent command/flag/env-var mapping. Triggers: migrate CI/CD, migrate + pipeline, kbc to kbagent, port GitHub Actions, CLI-based-sync-demo, kbc pull + push CI, project-as-code CI migration, replace kbc binary in CI, gitops + migration, multi-project promotion, dev to prod Keboola, KBC_STORAGE_API_TOKEN + to KBC_TOKEN, sync push CI, sync pull CI. +--- + +# kbc -> kbagent CI/CD Migration + +Guides a customer through porting a `kbc` GitHub CI/CD pipeline (the +[CLI-based-sync-demo](https://github.com/keboola/CLI-based-sync-demo) shape: +per-project pull/push, multi-project promotion, branch-gated deploys) to the new +`kbagent sync` engine, emitting **clean kbagent-native workflows**. + +## Reality check — this is a one-time BREAKING migration, not a command swap + +Do **not** tell the user they can just swap `kbc` for `kbagent` in place. Three +hard incompatibilities make this a deliberate cutover (verified against the code): + +1. **The on-disk layout/format is different and incompatible.** + - `kbc` writes per config: `config.json` + `meta.json` + `description.md` (JSON). + - `kbagent` writes per config: **`_config.yml`** (YAML, with `name`/`description`/ + `parameters` hoisted + a `_configuration_extra` block) + extracted code files + (`constants.py:425`, `sync/config_format.py`). + - The first `kbagent sync pull` therefore **rewrites every configuration** into a + new format. The old `config.json`/`meta.json` files are **not read** by kbagent + and become orphans that must be deleted. Expect a **massive reformatting diff**. +2. **kbagent sync is an ORCHESTRATOR, not cwd-per-folder.** `kbc pull` runs against + whatever directory you `cd` into. `kbagent sync pull` *requires* `--project ALIAS` + (resolved from a central config store) or `--all-projects` (`sync.py:67,495`). In + CI we bridge this with env-injection: `KBAGENT_PROJECT_FROM_ENV=1` synthesizes a + project under the reserved alias `__env__`, and every command passes + `--project __env__ --directory `. +3. **The two tools cannot co-own the same tree.** Because the source-of-truth files + differ, you cannot have `kbc push` and `kbagent push` both treating one directory + as canonical. You must cut over. + +### Consequence (this is what the user correctly anticipated) +- The migration is a **one-time conversion commit** on a **dedicated branch**, where + the JSON tree is replaced by the YAML tree. Reviewing that diff line-by-line is + impractical; you verify by **behavior** (`sync diff` clean, dry-run push empty), not + by reading the reformat. Merging it to `main` is effectively a tooling version bump. +- Until cutover, keep the legacy `kbc` workflows; afterwards delete them in the same PR. + +> **Live-verified (project 153, GCP europe-west3, 2026-06):** +> - `KBAGENT_PROJECT_FROM_ENV=1` + `--project __env__` works for `init` and `pull`. +> - kbc pulled **143 `config.json` + 187 `meta.json`** (JSON); kbagent pulled +> **147 `_config.yml`** (YAML). Zero overlap in file format. +> - `sync init --adopt-existing` on a **kbc-produced tree** succeeds, but the very +> next `sync diff` reports **`0 to create, 0 to update, 136 to delete`** — kbagent +> does not read kbc's `config.json` at all, so it sees every existing config as a +> local deletion. **A `sync push --allow-delete` here would delete all 136 remote +> configs.** Adopt-existing adopts only the manifest, NOT the configs. +> - The correct path — adopt → `sync pull --force` (writes `_config.yml`) → `git rm` +> the orphaned `config.json`/`meta.json` — converged the diff from 136 deletes to +> ~2 (plus ~9 remote-only scheduler/variables configs that pull/diff treat +> inconsistently — verify these per project). Both file sets coexist after pull +> until you delete the kbc files, so the `git rm` step is mandatory, not optional. +> +> **Operator rule:** never run `sync push` against an adopted-but-not-yet-pulled kbc +> tree. Always `sync pull --force` first, confirm `sync diff` is clean, THEN enable +> the push lane. + +### What still carries over unchanged +The *orchestration shell* is CLI-agnostic: manual/scheduled pull that commits state, +PR validation, GitHub-Environment-gated push, branch→env mapping, per-project loops. +Only three mechanics change: **install** (`uv tool install` not a binary download), +**commands/flags** (`kbagent sync ...`, see +[references/command-mapping.md](references/command-mapping.md)), and **auth env vars** +(`KBAGENT_PROJECT_FROM_ENV=1` + `KBC_TOKEN` + `KBC_STORAGE_API_URL`). + +## Workflow + +### Step 1 — Analyze the existing repo (always start here) +Run the engine in dry-run mode to inventory projects and the legacy CI it replaces: + +```bash +python /scripts/migrate_cicd.py /path/to/repo +``` + +It prints every project found (one per `.keboola/manifest.json`), each project's +id / stack host / required token secret name, any `ignoredComponents` ("subset of +a project" — see Step 5), and the legacy `kbc` workflow/action files it supersedes. + +### Step 2 — Pick a version pin (decide before generating) +- **Pinned (recommended for prod lanes):** `--version 0.58.0` (PyPI, once published) + or `--git-ref v0.58.0` (git tag, until PyPI exists). Reproducible CI. +- **Unpinned (`keboola-agent-cli`, resolves to latest):** only acceptable for a + non-prod/scratch lane. Warn the user: unpinned + the current auto-update behavior + means non-deterministic CI runs. + +### Step 3 — Generate the clean workflows +```bash +python /scripts/migrate_cicd.py /path/to/repo --write \ + --version 0.58.0 --main-branch main --schedule "0 * * * *" +``` +Produces: +- `.github/workflows/kbagent-validate.yml` — on PR: `sync diff` + `sync push --dry-run` per project (read-only drift + secret-encryption preflight). +- `.github/workflows/kbagent-pull.yml` — manual + optional cron: `sync pull --force` per project, commits state back. +- `.github/workflows/kbagent-push.yml` — manual, **GitHub-Environment-gated**: `sync push` per project, with an `allow_delete` input. + +The legacy `kbc` files are **left in place** — review the new ones, then delete the +old workflows/actions in the same PR. + +### Step 3b — Perform the one-time config conversion (dedicated branch) +This is the breaking part. On a fresh migration branch, for each project, convert +the JSON tree to kbagent's YAML tree and remove the orphaned kbc files: + +```bash +git checkout -b migrate/kbc-to-kbagent +# Per project (do ONE non-prod project first and verify): +KBAGENT_PROJECT_FROM_ENV=1 KBC_TOKEN=$L0_TOKEN KBC_STORAGE_API_URL=https://connection.keboola.com \ + kbagent sync init --adopt-existing --project __env__ --directory L0 +KBAGENT_PROJECT_FROM_ENV=1 KBC_TOKEN=$L0_TOKEN KBC_STORAGE_API_URL=https://connection.keboola.com \ + kbagent sync pull --project __env__ --directory L0 +# Remove orphaned kbc JSON files that kbagent no longer reads: +find L0 -name config.json -o -name meta.json | xargs git rm --cached --ignore-unmatch +git add -A +``` +Verify by **behavior**, not by reading the reformat diff: a follow-up +`sync diff --project __env__ -d L0` must be clean and `sync push --dry-run` empty. +Only then repeat for the remaining projects and the production lane. + +### Step 4 — Set up GitHub secrets, variables, environments +The engine prints exact `gh` commands. The model: **one Storage API token secret +per project** (`KBC_TOKEN_`), and two **Environments** (`prod`, `dev`) so +prod pushes require approval. See [references/secrets-setup.md](references/secrets-setup.md) +for the full mapping from the old `secrets.KBC_SAPI_TOKEN_*` / `vars.KBC_*` scheme. + +### Step 5 — Confirm scope ("subset of a project") and branching +- **Subset:** if a project should only sync part of its config tree, set + `ignoredComponents` (and/or `allowedBranches`) in that project's + `.keboola/manifest.json` — `kbagent sync` honors both, exactly like `kbc`. +- **Branching:** the old model used a fixed branch id per env. The new model maps + git branch → Keboola dev branch via `.keboola/branch-mapping.json` + + `kbagent sync branch-link`. For PR-based promotion this is usually *better*: + a PR branch links to a Keboola dev branch, `main` pushes to production. Add + `--git-branching` to annotate, and walk the user through `branch-link` if they + want per-PR isolated dev branches. If they want to keep the simple single-branch + (production) model, leave branch-mapping at the default (null = production). + +### Step 6 — Validate before merging +- Open the migration PR; the `kbagent-validate` workflow runs `sync diff` — confirm + the diff is empty (no unintended drift) against each project. +- Manually run `kbagent-pull` once and confirm the committed state matches what + `kbc pull` produced (the layout is identical; `git diff` should be tiny — mostly + YAML vs JSON config-body formatting differences if any). +- Do a `kbagent-push` dry-run (the validate workflow already does this) and read + the planned changes before the first real gated push. + +## Guardrails (state these to the user) +- **Never** add `--allow-plaintext-on-encrypt-failure` to CI push — it silently + uploads `#`-secrets in cleartext if the Encryption API is down. The generated + push is fail-closed by design. +- `sync push --allow-delete` deletes remote configs removed locally. It is wired to + the `allow_delete` workflow input (default off). Treat it like the old `--force`. +- Tokens live **only** in GitHub secrets and are injected as env vars per step; the + generated workflows never write a `config.json` to disk. + +## Reference material +- [references/migration-runbook.md](references/migration-runbook.md) — **the ordered PR sequence / cutover plan** (pre-flight → conversion PR → start-over). Use this when the user asks "which PRs, what order, how do I cut over." +- [references/branching-model.md](references/branching-model.md) — **how to choose** single-branch (Model A) vs git-branching (Model B), with a decision table. +- [references/command-mapping.md](references/command-mapping.md) — kbc ↔ kbagent commands, flags, env vars. +- [references/secrets-setup.md](references/secrets-setup.md) — secrets/vars/environments migration table + `gh` setup. +- `scripts/migrate_cicd.py` — the analyzer + generator (stdlib only). diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/references/branching-model.md b/plugins/kbagent/skills/kbagent-cicd-migration/references/branching-model.md new file mode 100644 index 00000000..65d02942 --- /dev/null +++ b/plugins/kbagent/skills/kbagent-cicd-migration/references/branching-model.md @@ -0,0 +1,67 @@ +# Choosing a branching model + +kbagent supports two ways to map your git repo to Keboola branches. Pick one up +front — it changes how `sync init` is run and what `push` touches. + +## The two models + +### Model A — Single-branch / production-direct +- Each project directory maps to **one Keboola project's production branch**. +- `.keboola/branch-mapping.json` stays at its default (`null` = production). +- `sync pull` / `sync push` read and write that project's production branch directly. +- Promotion across environments (dev → prod project) is a **git merge** between + project directories/repos, then a `push` to the next project (the demo's L0→L1 + shape). + +**Choose A when:** +- You're experimenting, or driving a single dev project locally instead of kbc. +- Your "environments" are *separate Keboola projects* (L0/L1), not dev branches. +- You want the simplest mental model and fewest moving parts. + +**Trade-off:** a `push` writes straight to the production branch of that project — +there is no isolated staging copy inside Keboola. Review happens in git, not in KBC. + +### Model B — Git-branching (Keboola dev-branch isolation) +- `sync init --git-branching` creates `.keboola/branch-mapping.json`. +- Each **git branch** links to a **Keboola development branch** (an isolated server- + side copy) via `kbagent sync branch-link --branch-name `. +- Work on a PR branch → `push` lands in its Keboola dev branch (safe, isolated); + merge to `main` → `push` lands in production. +- `kbagent sync branch-status` shows the mapping; `branch-unlink` detaches. + +**Choose B when:** +- Multiple people open PRs against the same project and you want each change tested + in isolation inside Keboola before it hits production. +- You already use Keboola's development-branches feature. +- You want PRs to never write production directly. + +**Trade-off:** more lifecycle to manage (create/link/unlink dev branches, clean them +up), and the mapping file is per-clone state. + +## Decision shortcut + +| Your situation | Model | +|---|---| +| "I just want to manipulate one project with kbagent instead of kbc" | **A** (start here) | +| Separate dev/prod **projects** promoted by git merge (L0/L1) | **A** | +| PR-per-change, multiple contributors, want isolated server-side testing | **B** | +| You rely on Keboola development branches today | **B** | + +You can start on **A** and adopt **B** later: run `sync init --git-branching` and +`branch-link` when you actually need per-PR isolation. Moving A→B is additive (it adds +a mapping file); it does not require re-converting the config tree. + +## How the model shows up in commands + +```bash +# Model A (production-direct) — nothing special: +kbagent sync pull --project -d +kbagent sync push --project -d + +# Model B (git-branching): +kbagent sync init --git-branching --project -d +git checkout -b feature/x +kbagent sync branch-link --project -d --branch-name feature/x +kbagent sync pull/push --project -d # now targets the dev branch +kbagent sync branch-status --project -d +``` diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md b/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md new file mode 100644 index 00000000..7fa4e2f1 --- /dev/null +++ b/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md @@ -0,0 +1,59 @@ +# kbc ↔ kbagent command / flag / env mapping + +Authoritative mapping used by the migration generator. Verify flags against your +installed `kbagent` version (`kbagent sync pull --help`); the new CLI evolves fast. + +## Install + +| kbc (old) | kbagent (new) | +|---|---| +| Download Go binary zip from `keboola/keboola-as-code` GitHub release, unzip to `/usr/local/bin/kbc` | `uv tool install keboola-agent-cli==` (PyPI) or `uv tool install 'git+https://github.com/keboola/cli@'` | +| `kbc --version` | `kbagent version` | +| Custom `install` composite action | `astral-sh/setup-uv@v5` + one `uv tool install` line | + +## Core sync commands + +| kbc (old) | kbagent (new) | Notes | +|---|---|---| +| `kbc init -d DIR --allow-target-env` | `kbagent sync init --directory DIR [--adopt-manifest]` | `--adopt-manifest` reuses an existing `.keboola/manifest.json` written by `kbc` | +| `kbc persist -d DIR` | *(folded into `sync pull`)* | No separate persist step; pull writes manifest + new objects | +| `kbc pull -d DIR --force` | `kbagent sync pull --directory DIR --force` | `--force` overrides local-vs-remote conflicts (3-way diff) | +| `kbc push -d DIR` | `kbagent sync push --directory DIR` | Encrypts `#`-secrets fail-closed before write | +| `kbc push -d DIR --force` | `kbagent sync push --directory DIR --allow-delete` | `--allow-delete` removes remote configs deleted locally | +| `kbc push --dry-run` / push-dry action | `kbagent sync push --dry-run --directory DIR` | Shows planned changes without writing | +| `kbc diff -d DIR` | `kbagent sync diff --directory DIR [--json]` | `--json` gives structured drift for CI gating | +| `kbc status` | `kbagent sync status --directory DIR` | | +| `kbc validate` (JSON-schema) | *(no direct equivalent — gap)* | Use `sync diff` for drift; schema validation is not ported | + +## Auth / environment variables + +| kbc (old) | kbagent (new) | Notes | +|---|---|---| +| `KBC_STORAGE_API_TOKEN` | `KBC_TOKEN` | Storage API token | +| `KBC_STORAGE_API_HOST` (bare host) | `KBC_STORAGE_API_URL` (full URL) | `connection.keboola.com` → `https://connection.keboola.com` | +| *(implicit)* | `KBAGENT_PROJECT_FROM_ENV=1` | **Required** opt-in so kbagent synthesizes an ephemeral project from the env in CI (no `config.json` on disk). See `constants.py:163`, `config_store.py:193` | +| `KBC_PROJECT_ID`, `KBC_BRANCH_ID`, `KBC_BRANCHES` | *(from manifest + branch-mapping)* | Project id comes from `.keboola/manifest.json`; branch from `.keboola/branch-mapping.json` | + +## Branching + +| kbc (old) | kbagent (new) | +|---|---| +| Fixed `KBC_BRANCH_ID` per env; `allowedBranches` in manifest | `.keboola/branch-mapping.json` (git branch → Keboola branch id; `null` = production) managed by `kbagent sync branch-link / branch-unlink / branch-status` | +| Branch dir under repo (`main/`) | Same on-disk layout; mapping decides which Keboola branch a git branch targets | + +## Subset of a project + +Both CLIs honor manifest-level scoping — no command change needed: + +- `allowedBranches: [""]` — restrict which branches sync. +- `ignoredComponents: ["keboola.foo", ...]` — exclude component types. + +`kbagent` parses both (`sync/manifest.py:120`). Additionally, `sync pull` flags +`--skip-storage` / `--skip-jobs` / `--with-table-samples` control how much +*metadata* (beyond configs) is pulled — orthogonal to the config subset. + +## What has NO clean port (call out to the user) +- `kbc validate` JSON-schema validation. +- `kbc ci workflows` generator itself (this skill replaces it). +- Templates / dbt / CI-scaffold subsystems (`kbc template`, `kbc dbt`) — keep `kbc` + for those; they are out of scope for sync CI/CD. diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md b/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md new file mode 100644 index 00000000..cacfa695 --- /dev/null +++ b/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md @@ -0,0 +1,93 @@ +# Migration runbook — kbc → kbagent (PR sequence) + +The ordered, low-risk way to cut a repo over. This is a **clean cutover**, not a +coexistence: kbc (`config.json`/`meta.json`) and kbagent (`_config.yml`) cannot both +own the same tree (live-verified — see the SKILL.md reality note). Plan it as one +conversion PR plus housekeeping, then everyone re-branches from the new `main`. + +## Answer to "can I transition seamlessly to a new branch?" +No co-existence, but yes a controlled cutover: +1. One **conversion PR** flips the whole repo from JSON→YAML + swaps the workflows. +2. Merge it to `main`. +3. **Delete/redo every old branch** — they carry the incompatible kbc layout and can + never cleanly merge into the converted `main`. +4. Everyone branches fresh from the new `main` with the new workflows. + +## Pre-flight (do once, before any PR) +- [ ] **Announce a change freeze** on the repo + the Keboola projects for the + conversion window. Any config edit made in the UI between "pull" and "cutover" + becomes drift you'll chase. Keep it short. +- [ ] **Pick a kbagent version** and pin it (`keboola-agent-cli==X.Y.Z` or + `git+...@vX.Y.Z`). Never unpinned on a prod lane. +- [ ] **Set GitHub secrets**: one `KBC_TOKEN_` per project (see + `references/secrets-setup.md`). +- [ ] **Create GitHub Environments** `dev` + `prod`; add required reviewers to `prod`. +- [ ] **Inventory** with the skill's analyzer (dry-run): confirm every project and the + legacy files it will replace. + `python /scripts/migrate_cicd.py /path/to/repo` + +## PR 1 — Conversion (the big one) → branch `migrate/kbc-to-kbagent` +Do the projects one at a time; **start with a non-prod project (e.g. `dev`/`L0`)**. + +Per project `` (with its token in the env): +```bash +export KBAGENT_PROJECT_FROM_ENV=1 KBC_TOKEN=$TOKEN \ + KBC_STORAGE_API_URL=https:// +kbagent sync init --adopt-existing --project __env__ --directory +kbagent sync pull --force --project __env__ --directory # writes _config.yml +# Drop the orphaned kbc files kbagent does not read: +find \( -name config.json -o -name meta.json \) -exec git rm -q {} + +# VERIFY BY BEHAVIOR (must be clean before you trust the project): +kbagent sync diff --project __env__ --directory +``` +Acceptance for each project: `sync diff` shows `0 to create, 0 to update, 0 to delete`. +- If a handful of **scheduler / variables** configs show as "to create" (a known + wrinkle from the live test), pull again and confirm they settle; if they persist, + note them in the PR and reconcile manually before enabling push. + +Then, still on the same branch: +```bash +# Generate the clean kbagent-native workflows: +python /scripts/migrate_cicd.py /path/to/repo --write --version X.Y.Z +# Remove the legacy kbc CI (the analyzer listed these): +git rm -r .github/actions/kbc_* .github/workflows/KBC_*.yml # adjust to your repo +git add -A && git commit -m "Migrate kbc -> kbagent: convert configs + workflows" +``` + +**Review this PR by behavior, not by diff.** The reformat touches hundreds of files; +reading it line-by-line is pointless. Trust: +- the `kbagent-validate` workflow runs on the PR and `sync diff` is clean per project; +- a `sync push --dry-run` (also in validate) reports no changes. + +Merge to `main` once validate is green. + +## PR 2 — Housekeeping (optional, after merge) +- [ ] Branch protection on `main`; require the `kbagent-validate` check. +- [ ] Tune the pull schedule cron / push approval reviewers. +- [ ] Update the repo README to the new install + commands. +- [ ] Decide the branching model (next section). + +## After merge — start over +- [ ] **Close or recreate every open PR** that was based on the kbc layout. They diff + against JSON files that no longer exist; rebasing them is not worth it — redo the + change on a fresh branch from the converted `main`. +- [ ] **Delete stale feature branches** (`git push origin --delete `). +- [ ] Tell contributors to **re-clone or hard-reset** to the new `main`. +- [ ] First real push: run `kbagent push` (workflow_dispatch) to `dev` first, approve, + verify in the Keboola UI, then to `prod`. + +## Branching model — pick one +| Model | When | How | +|---|---|---| +| **Single-branch (production)** | Each git branch/project maps straight to a production Keboola project (the demo's L0/L1 promotion) | Leave `.keboola/branch-mapping.json` at default (null = production); gate prod pushes via the GitHub Environment | +| **Git-branching (dev isolation)** | You want each PR to deploy to an isolated Keboola dev branch, then merge to prod on merge-to-main | `kbagent sync init --git-branching`; `kbagent sync branch-link --project __env__ --branch-name ` in the PR workflow; `main` maps to production | + +For most teams already doing PR-per-change promotion, **git-branching** is the closer +fit and is safer (no direct prod writes from PRs). Migrate to it in PR 2, not PR 1. + +## Hard guardrails (repeat to the user) +- **Never** `kbagent sync push` against an adopted-but-not-yet-pulled kbc tree — it + reports every config as "to delete" (136 in the live test) and `--allow-delete` + would wipe the project. +- **Never** `--allow-plaintext-on-encrypt-failure` in CI. +- Keep the change freeze until `main` is converted and the first dev push is verified. diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/references/secrets-setup.md b/plugins/kbagent/skills/kbagent-cicd-migration/references/secrets-setup.md new file mode 100644 index 00000000..5b07a97f --- /dev/null +++ b/plugins/kbagent/skills/kbagent-cicd-migration/references/secrets-setup.md @@ -0,0 +1,52 @@ +# GitHub secrets / variables / environments setup + +The legacy CLI-based-sync-demo split config across **repo secrets**, **repo +variables**, and **GitHub Environments**. The kbagent model is simpler: one token +secret per project, the stack URL baked from each manifest, environments only for +push approval. + +## Migration table + +| Legacy (kbc demo) | Type | kbagent (new) | Type | Notes | +|---|---|---|---|---| +| `secrets.KBC_SAPI_TOKEN_L0` | secret | `secrets.KBC_TOKEN_L0` | secret | One per project; injected as `KBC_TOKEN` | +| `secrets.KBC_SAPI_TOKEN_L1` | secret | `secrets.KBC_TOKEN_L1` | secret | | +| `vars.KBC_SAPI_HOST` | variable | *(baked from manifest `apiHost`)* | — | Override per project in the generated `env:` block if you use a non-default stack | +| `vars.KBC_PROJECT_ID_L0/L1` | variable | *(from `.keboola/manifest.json`)* | — | No longer a CI variable | +| `vars.KBC_BRANCH_ID_L0/L1` | variable | *(from `.keboola/branch-mapping.json`)* | — | Only if using git-branching mode | +| Environments `prod` / `dev` | environment | Environments `prod` / `dev` | environment | **Keep** — used for push approval gating | + +## Setup with `gh` + +```bash +REPO=/ + +# One Storage API token per project (use environment-scoped secrets for prod): +gh secret set KBC_TOKEN_L0 --repo "$REPO" # paste project 9996 token +gh secret set KBC_TOKEN_L1 --repo "$REPO" # paste project 9997 token + +# Environments for approval gating: +gh api -X PUT "repos/$REPO/environments/dev" +gh api -X PUT "repos/$REPO/environments/prod" +``` + +Then in the GitHub UI (or via the environments API): +1. Scope `KBC_TOKEN_*` for production projects to the **prod** environment. +2. Add **required reviewers** to the `prod` environment so `kbagent push` to prod + blocks on manual approval (this replaces the demo's environment gating). +3. Optionally restrict the `prod` environment to the `main` branch. + +## Why no token in config.json +`kbagent` can read a committed `.kbagent/config.json` with multiple project +aliases, but that file stores tokens — unsafe to commit. In CI we instead set +`KBAGENT_PROJECT_FROM_ENV=1` + `KBC_TOKEN` + `KBC_STORAGE_API_URL` per step, so the +token exists only as a masked GitHub secret in the runner's env, never on disk. +This is the direct, safer analog of the demo's per-project `KBC_SAPI_TOKEN_*` +secret model. + +## Security guardrails +- Do **not** commit `.kbagent/config.json` with tokens (the new CLI auto-writes a + `.gitignore` for its config dir — `config_store.py:359`). +- Do **not** pass `--allow-plaintext-on-encrypt-failure` in CI. +- Prefer environment-scoped secrets + required reviewers for any lane that pushes + to a production project. diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py new file mode 100755 index 00000000..70d08f64 --- /dev/null +++ b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python3 +"""Migrate a kbc (keboola-as-code) GitHub CI/CD repo to kbagent (keboola-agent-cli). + +This is the engine the ``kbagent-cicd-migration`` skill drives. It: + + 1. Discovers every Keboola project in the repo by locating ``.keboola/manifest.json`` + files (supports the multi-project layout, e.g. ``L0/``, ``L1/``). + 2. Reads each manifest's ``project.id`` / ``project.apiHost`` / + ``allowedBranches`` / ``ignoredComponents`` so generated workflows are + project-accurate and the "subset of a project" lever is surfaced. + 3. Detects the legacy kbc CI/CD it will replace (``.github/workflows`` + + ``.github/actions`` referencing ``kbc``). + 4. Emits **clean kbagent-native** GitHub Actions workflows + (validate / pull / push) that use ``uv tool install`` + ``kbagent sync`` + with the env-injection auth model — no per-CLI composite actions, no + committed tokens. + 5. Prints the exact GitHub **secrets + variables + environments** the new + workflows need, with copy-paste ``gh`` CLI commands. + +Stdlib only. Dry-run by default; pass ``--write`` to write files. + +Usage: + python migrate_cicd.py [--write] \\ + [--version 0.58.0 | --git-ref vX.Y.Z] \\ + [--main-branch main] [--schedule "0 * * * *"] \\ + [--git-branching] + +Examples: + # Inspect what would change (no writes): + python migrate_cicd.py ../CLI-based-sync-demo + + # Generate workflows pinned to a published PyPI version: + python migrate_cicd.py ../CLI-based-sync-demo --write --version 0.58.0 + + # Pin to a git tag instead (no PyPI release yet): + python migrate_cicd.py ../CLI-based-sync-demo --write --git-ref v0.58.0 +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path + +# --------------------------------------------------------------------------- # +# Project discovery +# --------------------------------------------------------------------------- # + + +@dataclass +class Project: + """A single Keboola project discovered via its manifest.""" + + alias: str # derived from the directory name, uppercased for secret naming + directory: str # path relative to repo root (".", "L0", "L1", ...) + project_id: str + api_host: str + allowed_branches: list[str] = field(default_factory=list) + ignored_components: list[str] = field(default_factory=list) + + @property + def token_secret(self) -> str: + return f"KBC_TOKEN_{self.alias}" + + @property + def stack_url(self) -> str: + # apiHost in the manifest is bare ("connection.keboola.com"); kbagent's + # KBC_STORAGE_API_URL wants a full URL. + host = self.api_host.strip() + if host.startswith(("http://", "https://")): + return host + return f"https://{host}" + + +def _alias_from_dir(directory: str) -> str: + name = Path(directory).name or "PROJECT" + return re.sub(r"[^A-Za-z0-9]+", "_", name).strip("_").upper() or "PROJECT" + + +def discover_projects(repo: Path) -> list[Project]: + """Find every ``.keboola/manifest.json`` and parse it into a Project.""" + projects: list[Project] = [] + for manifest_path in sorted(repo.glob("**/.keboola/manifest.json")): + project_dir = manifest_path.parent.parent + rel = project_dir.relative_to(repo).as_posix() + rel = "." if rel == "" else rel + try: + data = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + print(f" ! skipping {manifest_path}: {exc}", file=sys.stderr) + continue + proj = data.get("project", {}) + projects.append( + Project( + alias=_alias_from_dir(rel), + directory=rel, + project_id=str(proj.get("id", "")), + api_host=str(proj.get("apiHost", "")), + allowed_branches=[str(b) for b in data.get("allowedBranches", [])], + ignored_components=[str(c) for c in data.get("ignoredComponents", [])], + ) + ) + return projects + + +def detect_legacy_ci(repo: Path) -> list[str]: + """Return a list of legacy kbc CI/CD files that the migration supersedes.""" + found: list[str] = [] + gh = repo / ".github" + if not gh.exists(): + return found + for path in sorted(gh.glob("**/*")): + if not path.is_file() or path.suffix not in {".yml", ".yaml"}: + continue + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + continue + # A legacy file is one that invokes the kbc binary or its env vars. + if re.search(r"\bkbc\s+(pull|push|init|persist|diff|validate)\b", text) or ( + "KBC_STORAGE_API_TOKEN" in text + ): + found.append(path.relative_to(repo).as_posix()) + return found + + +# --------------------------------------------------------------------------- # +# Workflow generation (clean kbagent-native) +# --------------------------------------------------------------------------- # + + +def _install_steps(version: str | None, git_ref: str | None) -> str: + if git_ref: + spec = f"git+https://github.com/keboola/cli@{git_ref}" + elif version: + spec = f"keboola-agent-cli=={version}" + else: + # Unpinned: only acceptable for non-production lanes. The skill warns. + spec = "keboola-agent-cli" + return ( + " - name: Install uv\n" + " uses: astral-sh/setup-uv@v5\n" + " - name: Install kbagent\n" + f" run: uv tool install '{spec}'\n" + " - name: Show version\n" + " run: kbagent version\n" + ) + + +def _project_step(p: Project, command: str, step_name: str) -> str: + """Render one per-project step using the env-injection auth model. + + kbagent sync is an orchestrator over registered project aliases, NOT a + cwd-per-folder tool like kbc. In CI we synthesize an ephemeral project from + the env (KBAGENT_PROJECT_FROM_ENV=1 -> reserved alias ``__env__``) and pass + ``--project __env__`` explicitly. A first idempotent ``init --adopt-existing`` + registers the committed manifest before the real command. + """ + return ( + f" - name: {step_name} ({p.alias})\n" + " env:\n" + ' KBAGENT_PROJECT_FROM_ENV: "1"\n' + f" KBC_TOKEN: ${{{{ secrets.{p.token_secret} }}}}\n" + f" KBC_STORAGE_API_URL: {p.stack_url}\n" + " run: |\n" + f" kbagent sync init --adopt-existing --project __env__ --directory '{p.directory}' || true\n" + f" kbagent sync {command} --project __env__ --directory '{p.directory}'\n" + ) + + +def gen_validate(projects: list[Project], main_branch: str) -> str: + diff_steps = "".join(_project_step(p, "diff --json", f"Diff {p.directory}") for p in projects) + dry_run_steps = "".join( + _project_step(p, "push --dry-run", f"Push dry-run {p.directory}") for p in projects + ) + return ( + "# Generated by kbagent-cicd-migration. Clean kbagent-native CI.\n" + "name: kbagent validate\n" + "on:\n" + " pull_request:\n" + " workflow_dispatch:\n" + "permissions:\n" + " contents: read\n" + "jobs:\n" + " validate:\n" + " runs-on: ubuntu-latest\n" + " steps:\n" + " - uses: actions/checkout@v4\n" + f"{_install_steps_placeholder()}" + " # Show drift between the committed config files and each remote\n" + " # project. `sync diff` is read-only. The push dry-run below also\n" + " # surfaces secret-encryption problems before a real push.\n" + f"{diff_steps}" + f"{dry_run_steps}" + ) + + +def gen_pull(projects: list[Project], main_branch: str, schedule: str | None) -> str: + on_block = " workflow_dispatch:\n" + if schedule: + on_block += f" schedule:\n - cron: '{schedule}'\n" + steps = "".join(_project_step(p, "pull --force", f"Pull {p.directory}") for p in projects) + return ( + "# Generated by kbagent-cicd-migration. Pulls remote state into git.\n" + "name: kbagent pull\n" + "on:\n" + f"{on_block}" + "permissions:\n" + " contents: write\n" + "jobs:\n" + " pull:\n" + " runs-on: ubuntu-latest\n" + " steps:\n" + " - uses: actions/checkout@v4\n" + f"{_install_steps_placeholder()}" + f"{steps}" + " - name: Commit pulled state\n" + " run: |\n" + " git config user.name 'Keboola kbagent'\n" + " git config user.email 'kbagent@users.noreply.github.com'\n" + " git add -A\n" + " git commit -m \"Automatic kbagent pull $(date -u +%Y-%m-%dT%H:%M:%SZ)\" || echo 'no changes'\n" + " git push\n" + ) + + +def gen_push(projects: list[Project], main_branch: str, git_branching: bool) -> str: + # GitHub Environments gate production approvals; the prod environment maps to + # the main branch, mirroring the legacy `github.ref_name == 'main'` logic. + env_expr = f"${{{{ github.ref_name == '{main_branch}' && 'prod' || 'dev' }}}}" + # The workflow_dispatch boolean arrives as the string 'true'/'false'; the GH + # expression maps it to the --allow-delete flag (opt-in deletion of remote + # configs that were removed locally). + delete_expr = "${{ github.event.inputs.allow_delete == 'true' && '--allow-delete' || '' }}" + steps = "".join( + _project_step(p, f"push {delete_expr}", f"Push {p.directory}") for p in projects + ) + return ( + "# Generated by kbagent-cicd-migration. Pushes git state to Keboola.\n" + "# Protected by a GitHub Environment so prod pushes require approval.\n" + "name: kbagent push\n" + "on:\n" + " workflow_dispatch:\n" + " inputs:\n" + " allow_delete:\n" + " description: 'Delete remote configs removed locally'\n" + " type: boolean\n" + " default: false\n" + "permissions:\n" + " contents: read\n" + "jobs:\n" + " push:\n" + f" environment: {env_expr}\n" + " runs-on: ubuntu-latest\n" + " steps:\n" + " - uses: actions/checkout@v4\n" + f"{_install_steps_placeholder()}" + " # `sync push` encrypts #-secrets fail-closed by default. Do NOT add\n" + " # --allow-plaintext-on-encrypt-failure in CI.\n" + f"{steps}" + ) + + +# Install steps are injected after generation so the version/ref is applied once. +_INSTALL_TOKEN = "@@INSTALL@@\n" + + +def _install_steps_placeholder() -> str: + return _INSTALL_TOKEN + + +# --------------------------------------------------------------------------- # +# Secrets / variables checklist +# --------------------------------------------------------------------------- # + + +def secrets_report(projects: list[Project], repo_slug: str) -> str: + lines: list[str] = [] + lines.append("Required GitHub secrets (per project Storage API token):") + for p in projects: + lines.append( + f" gh secret set {p.token_secret} " + f"--repo {repo_slug} # project {p.project_id} ({p.directory})" + ) + lines.append("") + lines.append("Required GitHub Environments (for `kbagent push` approval gating):") + lines.append(f" gh api -X PUT repos/{repo_slug}/environments/prod") + lines.append(f" gh api -X PUT repos/{repo_slug}/environments/dev") + lines.append( + " # Then scope each KBC_TOKEN_* secret to its environment and add " + "required reviewers to 'prod' in the GitHub UI." + ) + lines.append("") + lines.append( + "No KBC_STORAGE_API_URL secret needed: it is baked from each manifest's " + "apiHost. Override per project by editing the generated env: block." + ) + return "\n".join(lines) + + +# --------------------------------------------------------------------------- # +# Orchestration +# --------------------------------------------------------------------------- # + + +def _guess_repo_slug(repo: Path) -> str: + config = repo / ".git" / "config" + if config.exists(): + m = re.search(r"github\.com[:/]([^/]+/[^/\s.]+)", config.read_text(errors="ignore")) + if m: + return m.group(1) + return "/" + + +def run(args: argparse.Namespace) -> int: + repo = Path(args.repo_dir).resolve() + if not repo.is_dir(): + print(f"error: {repo} is not a directory", file=sys.stderr) + return 2 + + projects = discover_projects(repo) + if not projects: + print( + f"error: no .keboola/manifest.json found under {repo}. " + "Is this a kbc project-as-code repo?", + file=sys.stderr, + ) + return 2 + + print(f"Discovered {len(projects)} project(s) in {repo}:") + for p in projects: + subset = "" + if p.ignored_components: + subset = f" [subset: {len(p.ignored_components)} ignored component(s)]" + print( + f" - {p.directory:<8} id={p.project_id:<8} host={p.api_host}" + f" token=secrets.{p.token_secret}{subset}" + ) + + legacy = detect_legacy_ci(repo) + print(f"\nLegacy kbc CI/CD files detected ({len(legacy)}):") + for f in legacy: + print(f" - {f}") + if legacy: + print( + " NOTE: these are NOT deleted. Review the new workflows, then remove " + "the legacy ones in the same PR." + ) + + install = _install_steps(args.version, args.git_ref) + files = { + ".github/workflows/kbagent-validate.yml": gen_validate(projects, args.main_branch), + ".github/workflows/kbagent-pull.yml": gen_pull(projects, args.main_branch, args.schedule), + ".github/workflows/kbagent-push.yml": gen_push( + projects, args.main_branch, args.git_branching + ), + } + files = {k: v.replace(_INSTALL_TOKEN, install) for k, v in files.items()} + + print(f"\nGenerated workflows ({'WRITING' if args.write else 'dry-run, use --write'}):") + for rel, content in files.items(): + target = repo / rel + print(f" - {rel} ({len(content.splitlines())} lines)") + if args.write: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + + print("\n" + "=" * 70) + print(secrets_report(projects, _guess_repo_slug(repo))) + print("=" * 70) + + if not args.write: + print("\nDry-run only. Re-run with --write to create the files above.") + return 0 + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("repo_dir", help="Path to the kbc project-as-code repo to migrate") + ap.add_argument( + "--write", action="store_true", help="Write the generated workflows (default: dry-run)" + ) + grp = ap.add_mutually_exclusive_group() + grp.add_argument("--version", help="Pin kbagent to this PyPI version, e.g. 0.58.0") + grp.add_argument("--git-ref", help="Pin kbagent to a git tag/ref, e.g. v0.58.0 (no PyPI yet)") + ap.add_argument( + "--main-branch", + default="main", + help="Branch that maps to the prod environment (default: main)", + ) + ap.add_argument( + "--schedule", default=None, help="Cron for scheduled pull, e.g. '0 * * * *' (default: none)" + ) + ap.add_argument( + "--git-branching", action="store_true", help="Annotate for git-branch->Keboola-branch mode" + ) + return run(ap.parse_args(argv)) + + +if __name__ == "__main__": + raise SystemExit(main()) From 2e1f5f85d98fc3c0e03dbcb2e1a0d70fce1c5dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 6 Aug 2026 12:16:16 +0200 Subject: [PATCH 03/12] fix(skill): live-verify kbc->kbagent migration mechanics, fix push flag Ran the recommended migration path end-to-end against project 153 (fresh kbc init pull + kbagent v0.80.0) to verify the skill's own claims: - migrate_cicd.py generated a `push` workflow using a nonexistent `--allow-delete` flag; kbagent's actual flag is `--force`. Fixed the generator and the command-mapping table. - Plain `sync init` (the recommended no-adopt-existing mechanic) cannot run verbatim against a directory straight out of `kbc pull` -- kbc and kbagent share the same manifest path (`.keboola/manifest.json`), so it fails fast with "Manifest already exists" until that one file (not the config.json/meta.json tree) is deleted first. Folded the missing `rm` step into SKILL.md and migration-runbook.md. - The "delete now-empty kbc-only type folders" cleanup step was a no-op: app/processor/_shared still hold description.md + code bodies kbagent never reads, so `find -empty -delete` matches nothing. Replaced with a full subtree removal, confirmed to leave `sync diff`/`sync status` unaffected. - Re-confirmed the `--adopt-existing` phantom-rows bug reproduces exactly as documented (9 added / 1 deleted) on a side-by-side copy of the same tree, validating the plain-init recommendation. - Added a Prerequisites section (repo path, kbagent install, per-project storage host+token, which project first) that the skill previously assumed rather than stated. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../skills/kbagent-cicd-migration/SKILL.md | 230 +++++++++++++++++- .../references/command-mapping.md | 4 +- .../references/migration-runbook.md | 100 +++++++- .../scripts/migrate_cicd.py | 7 +- 4 files changed, 316 insertions(+), 25 deletions(-) diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md index 78b2240b..e7655e99 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md @@ -57,7 +57,7 @@ hard incompatibilities make this a deliberate cutover (verified against the code > - `sync init --adopt-existing` on a **kbc-produced tree** succeeds, but the very > next `sync diff` reports **`0 to create, 0 to update, 136 to delete`** — kbagent > does not read kbc's `config.json` at all, so it sees every existing config as a -> local deletion. **A `sync push --allow-delete` here would delete all 136 remote +> local deletion. **A `sync push --force` here would delete all 136 remote > configs.** Adopt-existing adopts only the manifest, NOT the configs. > - The correct path — adopt → `sync pull --force` (writes `_config.yml`) → `git rm` > the orphaned `config.json`/`meta.json` — converged the diff from 136 deletes to @@ -65,9 +65,92 @@ hard incompatibilities make this a deliberate cutover (verified against the code > inconsistently — verify these per project). Both file sets coexist after pull > until you delete the kbc files, so the `git rm` step is mandatory, not optional. > -> **Operator rule:** never run `sync push` against an adopted-but-not-yet-pulled kbc -> tree. Always `sync pull --force` first, confirm `sync diff` is clean, THEN enable -> the push lane. +> **Re-verified (same project 153, kbagent v0.80.0, 2026-08): the 136-delete footgun +> no longer reproduces.** kbagent fixed `adopt-existing` between the original test and +> v0.80.0. Current behavior on a fresh `kbc`-produced tree (137 `config.json` + 181 +> `meta.json`): +> - `sync init --adopt-existing` + `sync diff` now reports +> `added: 0, deleted: 0, remote_only: 9, never_fetched: 110` — **no false deletes.** +> Configs kbagent hasn't pulled yet show as `never_fetched`, not `deleted`. +> - After `sync pull`, diff converges to `added: 9, deleted: 2, unchanged: 118` — a +> handful of genuine variable-row drift, not a mass delete. `sync push --dry-run` +> confirms: "would create 9, update 0, delete 2." +> - The orphaned-file claim **still holds**: 318 `config.json`/`meta.json` files +> coexisted with 146 new `_config.yml` files after pull. Deleting the orphans +> produced an *identical* `sync diff` summary before and after, confirming kbagent +> truly never reads them and the cleanup step is safe (though no longer +> safety-critical the way the delete-136 scenario was). +> +> **Takeaway:** re-verify the adopt/pull numbers against whichever kbagent version you +> are actually shipping — this mechanic has changed at least once. Don't quote the +> 136-delete figure as current behavior; it's a historical regression, not a standing +> hazard. The `git rm` orphan-cleanup step and the general "verify by behavior, not by +> reading the diff" guidance remain correct regardless of version. +> +> **Operator rule:** even though the mass-delete footgun is currently fixed, still +> never run `sync push` (without `--dry-run`) against an adopted-but-not-yet-pulled +> kbc tree. Always `sync pull` first, confirm `sync diff` is clean, THEN enable the +> push lane — a future regression or a customer on an older kbagent version could +> reintroduce the original failure mode. + +> **Root-caused (2026-08, project 153, kbagent v0.80.0): `--adopt-existing` never +> reaches a clean `sync status`, and DO NOT recommend it — use plain `sync init` +> instead (see "Recommended mechanic" below).** After adopt + pull, `sync status` +> still showed `added: 9, deleted: 1` even after the orphan-file cleanup. Traced to +> the manifest, not to remote drift: +> - `sync init --adopt-existing` carries kbc's row paths **verbatim** into the +> kbagent manifest — `values/{name}` for `keboola.variables`, `codes/{name}` for +> `keboola.shared-code` (kbc's own naming templates). A subsequent `sync pull` +> refreshes file *content* at an already-tracked row's existing path but never +> renames/relocates it to kbagent's own row convention. +> - kbagent's *own* row-path generator, exercised on a genuinely new row, always +> writes `rows/{name}` — confirmed by a side-by-side test: a **fresh** `sync init` +> (no adopt) + `sync pull` against the same project produced rows at +> `.../rows/default-values/_config.yml`, never `.../values/default/`. +> - `_find_untracked_configs` (the scanner behind both `sync status`'s "added" list +> and `sync push`'s create-plan) only excludes row files by checking for a literal +> `"rows"` path segment (`sync_service.py`). A row inherited at `values/...` or +> `codes/...` from adopt doesn't match that check, so **the scanner reports it as a +> brand-new top-level configuration**, with an empty `config_id` since it isn't +> actually one. **Pushing it would call `create_config` for a whole new sibling +> `keboola.variables`/`keboola.shared-code` configuration** (Phase A create path, +> not the row-update path), not update the row it actually is — a real duplicate- +> config risk on push, not just a cosmetic status mismatch. +> - Separately, kbc's own manifest.json stores **several unrelated companion +> `keboola.variables` configs at the same literal bare path `"variables"`**, +> resolving the true nesting only via a `relations` field kbagent's adopt path +> ignores entirely. Only one of those collides onto a real file after pull; the +> rest sit in the manifest with a stale `pull_hash` and no file at their recorded +> path → phantom `deleted` entries. This is a kbc-schema quirk (relation-based +> path resolution) that kbagent's manifest model has no equivalent for. +> - **A plain `sync init` (no `--adopt-existing`) + `sync pull` against the same +> project, verified twice (into an empty dir, and into a dir still containing kbc's +> untouched `config.json`/`meta.json` tree) produced `added: 0, modified: 0, +> deleted: 0, unchanged: 119` — a genuinely clean `sync status`.** Plain `init` +> ignores every foreign file it doesn't recognize; it only requires that +> `.keboola/manifest.json` not already exist. This is now the recommended +> mechanic — see below. +> +> **Re-verified end-to-end (2026-08-06, fresh `kbc init` pull of project 153 into +> an empty directory, kbc dev build + kbagent v0.80.0)**, side by side: +> - Plain `sync init` against the as-is kbc tree fails fast and by design: +> `Error: Manifest already exists at .../.keboola/manifest.json.` — because kbc +> and kbagent write to the identical path. `rm .keboola/manifest.json` (leaving +> every `config.json`/`meta.json` untouched), then plain `sync init` + `sync pull` +> reached `sync status` = "No local changes detected. (119 configurations tracked)" +> and `sync diff` = "No differences found." — genuinely, provably clean. +> - `sync init --adopt-existing` + `sync pull` against an identical untouched copy +> of the same tree reproduced the phantom-rows bug exactly as described above: +> `sync status` reported 9 added / 1 deleted, and `sync diff` planned 9 creates + +> 2 deletes — confirmed to be `keboola.variables`/`keboola.shared-code` rows +> inherited at kbc's `values/`/`codes/` paths, never relocated to kbagent's own +> `rows/` convention. +> - **Confirms the recommendation, but the write-up above was missing an +> operational step:** "plain init, no adopt-existing" cannot be run verbatim +> against a directory straight out of `kbc pull` — `.keboola/manifest.json` is +> always already there. The one-line fix, now folded into Step 3b below, is to +> `rm` that single file (never the `config.json`/`meta.json` tree it sits next +> to) immediately before the plain `init` call. ### What still carries over unchanged The *orchestration shell* is CLI-agnostic: manual/scheduled pull that commits state, @@ -77,6 +160,69 @@ Only three mechanics change: **install** (`uv tool install` not a binary downloa [references/command-mapping.md](references/command-mapping.md)), and **auth env vars** (`KBAGENT_PROJECT_FROM_ENV=1` + `KBC_TOKEN` + `KBC_STORAGE_API_URL`). +## Prerequisites — ask for these before Step 1 + +Four things this skill cannot infer; get them from the customer/operator first: + +- **The repo path.** Where the `kbc`-managed tree (with `.keboola/manifest.json` + and `.github/workflows/`) actually lives locally — Step 1's `migrate_cicd.py` + argument. +- **A `kbagent` binary or install.** Either already on `PATH`, or install it now: + `uv tool install keboola-agent-cli==` (see Step 2 for version pin) or a + downloaded standalone binary. No local `kbc` binary is required to *run* the + migration (kbagent is the only tool that touches the repo from Step 3b on) — + only to *verify* the "same data, new layout" claim by diffing a `kbc pull` + against a `kbagent sync pull` of the same project, which is optional. +- **Storage API host + token for each project being converted.** One pair per + project (`KBC_STORAGE_API_URL` + `KBC_TOKEN`, fed via + `KBAGENT_PROJECT_FROM_ENV=1`, see "auth env vars" above) — needed for every + `sync init`/`pull`/`diff`/`push` in Step 3b. Never ask the customer to paste a + token into chat; read it via the clipboard-secret pattern or point at wherever + they already store it (a registered `kbagent project`, a CI secret, a + password manager) and reference it by path/alias, not by value. +- **Which project to convert first.** Always non-prod — confirmed explicitly in + "How to run this" below, not assumed. + +## How to run this — ask the customer, don't auto-pilot + +This skill reads like a linear script, and it is tempting to run Steps 1-6 +end-to-end without stopping. **Don't.** Every step below that touches the +customer's repo or their live Keboola project is a decision the customer +should make, not one you make for them — this is their production CI/CD and +their production data. Treat the numbered steps as a checklist of decisions to +surface, not a batch job to execute. Concretely, stop and ask before you: + +- **Pick the version pin (Step 2).** Don't default to "latest" or to whatever + version happens to be installed on your machine — ask which lane (prod vs. + scratch) this repo is, and let the answer decide pinned vs. unpinned. +- **Write anything (`--write` in Step 3).** Show the dry-run output first, + let the customer review the projects/legacy-files inventory, then ask + before generating files into their repo. +- **Run `sync init` / `sync pull` against a real project (Step 3b).** This is + the breaking, one-time conversion — confirm which project to convert first + (always non-prod), and confirm the customer is fine with the change freeze + being in effect before you touch anything. Use plain `sync init` (no + `--adopt-existing`, see Step 3b) and require a genuinely empty `sync status` + before calling a project converted. +- **Delete any file** — orphaned `config.json`/`meta.json`, or the now-empty + kbc-only type folders (`app/`, `processor/`, `_shared/`, see the reality + check above). Show what you're about to delete and why (they are no longer + read by kbagent) and let the customer say go, especially the first time + through — don't `git rm` on their behalf and mention it after the fact. +- **Choose the branching model (Step 5).** Single-branch vs. git-branching is + a workflow/process decision for their team, not a default you pick because + it's "usually better." Present the decision table and wait for their answer. +- **Enable the push lane / run a real `sync push`.** Dry-run first, always; + a real push against a customer's project needs their explicit go-ahead + every time, not just once at the start of the migration. + +If you're running this yourself against a live project to verify the skill's +claims (as opposed to guiding a customer), the same discipline still applies: +narrate what you're about to run and why before you run it, rather than +chaining the whole sequence unattended — that's how stale/incorrect claims in +this skill (like the historical "136 to delete" figure) go unnoticed for two +months, and how you end up cleaning up things you didn't realize you'd need to. + ## Workflow ### Step 1 — Analyze the existing repo (always start here) @@ -112,23 +258,67 @@ old workflows/actions in the same PR. ### Step 3b — Perform the one-time config conversion (dedicated branch) This is the breaking part. On a fresh migration branch, for each project, convert -the JSON tree to kbagent's YAML tree and remove the orphaned kbc files: +the JSON tree to kbagent's YAML tree and remove the orphaned kbc files. + +**Recommended mechanic: plain `sync init` — do NOT use `--adopt-existing`.** +`--adopt-existing` carries kbc's row paths (`values/...`, `codes/...`) and its +`relations`-based companion-config paths straight into the kbagent manifest without +translating them to kbagent's own conventions. That's confirmed to leave a +permanently-dirty `sync status` (phantom `added`/`deleted` entries — see the +root-cause note above) and, worse, makes `sync push` create duplicate sibling +configs for the misclassified rows. Plain `init` has none of this baggage: it +creates a brand-new empty manifest and ignores every file it doesn't recognize +(kbc's `config.json`/`meta.json`/legacy folders included) — **except one**: kbc +and kbagent both write their manifest to the exact same path, `.keboola/manifest.json`. +Plain `sync init` refuses to run when that file already exists (`Error: Manifest +already exists at .../.keboola/manifest.json. Use 'sync pull' to update, 'sync +init --adopt-existing' to adopt a kbc-written manifest, or delete .keboola/ to +reinitialize.`) — confirmed live, this is not a hypothetical. **You must delete +kbc's manifest file (only that one file, not the `config.json`/`meta.json` config +tree) before plain `init` will run.** Once it's gone, pointing plain `init` at a +directory that still contains kbc's `config.json`/`meta.json` tree is safe — `pull` +then populates everything through kbagent's own naming/path logic from scratch. ```bash git checkout -b migrate/kbc-to-kbagent # Per project (do ONE non-prod project first and verify): +# NOTE: plain init, no --adopt-existing. kbc and kbagent share the same +# manifest path (.keboola/manifest.json), so plain init refuses to run +# until that one file is out of the way -- delete it first. +rm L0/.keboola/manifest.json KBAGENT_PROJECT_FROM_ENV=1 KBC_TOKEN=$L0_TOKEN KBC_STORAGE_API_URL=https://connection.keboola.com \ - kbagent sync init --adopt-existing --project __env__ --directory L0 + kbagent sync init --project __env__ --directory L0 KBAGENT_PROJECT_FROM_ENV=1 KBC_TOKEN=$L0_TOKEN KBC_STORAGE_API_URL=https://connection.keboola.com \ kbagent sync pull --project __env__ --directory L0 -# Remove orphaned kbc JSON files that kbagent no longer reads: -find L0 -name config.json -o -name meta.json | xargs git rm --cached --ignore-unmatch +# Remove orphaned kbc files that kbagent never reads: the config.json/meta.json +# tree, plus the kbc-only type folders (app/, processor/, _shared/) which also +# still hold description.md + code bodies kbagent never reads either -- these +# are NOT empty dirs, "find -empty" is a no-op against them (confirmed live); +# remove the whole subtree. See references/migration-runbook.md. +find L0 -name config.json -o -name meta.json | xargs git rm -q --ignore-unmatch +git rm -rq --ignore-unmatch L0/*/app L0/*/processor L0/*/_shared git add -A ``` Verify by **behavior**, not by reading the reformat diff: a follow-up -`sync diff --project __env__ -d L0` must be clean and `sync push --dry-run` empty. +`sync status --directory L0` **must show 0 added, 0 modified, 0 deleted** (not just +"a small handful" — plain init reaches a genuinely empty status; if it doesn't, +something is wrong and you should stop, not push through it), and +`sync diff --project __env__ -d L0` / `sync push --dry-run` must also be clean. Only then repeat for the remaining projects and the production lane. +**This is a normal commit on a normal branch — never a git-history rewrite.** +It is tempting, once you see how large the reformat diff is, to reach for +`git filter-repo` / an orphan-branch reset / a force-pushed squash of `main` to +make the history "clean." **Don't. This is not something you can deliver to a +real customer:** it invalidates every collaborator's existing clone and open PR, +destroys `git blame`/audit trail across the whole repo (a compliance problem for +regulated customers, not just an inconvenience), and requires a coordinated +force-push that most orgs' branch-protection rules block outright on `main` +anyway. The migration is disruptive enough as an ordinary large commit — do not +compound it with a history rewrite. Treat the "diff is huge, don't review it +line by line, verify by behavior instead" guidance above as the actual answer to +that discomfort, not a rewritten history. + ### Step 4 — Set up GitHub secrets, variables, environments The engine prints exact `gh` commands. The model: **one Storage API token secret per project** (`KBC_TOKEN_`), and two **Environments** (`prod`, `dev`) so @@ -160,10 +350,30 @@ for the full mapping from the old `secrets.KBC_SAPI_TOKEN_*` / `vars.KBC_*` sche - **Never** add `--allow-plaintext-on-encrypt-failure` to CI push — it silently uploads `#`-secrets in cleartext if the Encryption API is down. The generated push is fail-closed by design. -- `sync push --allow-delete` deletes remote configs removed locally. It is wired to +- `sync push --force` deletes remote configs removed locally. It is wired to the `allow_delete` workflow input (default off). Treat it like the old `--force`. - Tokens live **only** in GitHub secrets and are injected as env vars per step; the generated workflows never write a `config.json` to disk. +- **Never run `--all-projects` in a directory that also holds a flat single-project + tree.** `--all-projects` (`sync pull --all-projects` / `sync push --all-projects`) + is hard-coded to a `//` layout for *every* registered project + alias (`_sync_bulk.py`) — it is not "operate on whatever is in this directory." + Confirmed live: registering a persistent alias with `kbagent project add --project + 153 ...` (needed for ad-hoc `config update`/`config detail` maintenance work + outside the CI flow) and then running `sync pull --all-projects` in a directory + that already had a flat manifest at `./.keboola/manifest.json` silently + **auto-created a second, separate tree at `./153/`** — `pull_all` auto-inits any + registered alias with no manifest yet at its expected subpath, it does not detect + or reuse an existing flat-layout manifest for the same project. `push_all` is + slightly safer (skips instead of auto-creating) but still expects the same + subfolder convention. This is why the generated CI workflows never use + `--all-projects` — every step passes `--project __env__ --directory + '{directory}'` explicitly (see `migrate_cicd.py`'s `_project_step`). Carry the + same discipline into any manual/maintenance commands you run outside CI: always + `--project ALIAS --directory DIR` explicit, never `--all-projects`, in a + migration repo. If a customer (or you, helping them) already hit this, the + extra `/` directory can simply be deleted — it holds a fresh, unrelated + pull, not anything derived from their real tree. ## Reference material - [references/migration-runbook.md](references/migration-runbook.md) — **the ordered PR sequence / cutover plan** (pre-flight → conversion PR → start-over). Use this when the user asks "which PRs, what order, how do I cut over." diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md b/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md index 7fa4e2f1..d35d45a3 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md @@ -15,11 +15,11 @@ installed `kbagent` version (`kbagent sync pull --help`); the new CLI evolves fa | kbc (old) | kbagent (new) | Notes | |---|---|---| -| `kbc init -d DIR --allow-target-env` | `kbagent sync init --directory DIR [--adopt-manifest]` | `--adopt-manifest` reuses an existing `.keboola/manifest.json` written by `kbc` | +| `kbc init -d DIR --allow-target-env` | `rm DIR/.keboola/manifest.json && kbagent sync init --directory DIR` | For the one-time kbc→kbagent conversion use **plain `init`, not `--adopt-existing`** — adopting a kbc-written manifest inherits kbc's row/companion-config paths verbatim and leaves a permanently dirty `sync status` (root-caused; see SKILL.md and `migration-runbook.md`). kbc and kbagent both write to the same path (`.keboola/manifest.json`), so plain `init` errors "Manifest already exists" until you delete that one file (not the `config.json`/`meta.json` tree next to it) — confirmed live, 2026-08-06. `--adopt-existing` is still correct for re-registering an *already-converted* kbagent-native manifest in ephemeral CI (no kbc data involved at that point). | | `kbc persist -d DIR` | *(folded into `sync pull`)* | No separate persist step; pull writes manifest + new objects | | `kbc pull -d DIR --force` | `kbagent sync pull --directory DIR --force` | `--force` overrides local-vs-remote conflicts (3-way diff) | | `kbc push -d DIR` | `kbagent sync push --directory DIR` | Encrypts `#`-secrets fail-closed before write | -| `kbc push -d DIR --force` | `kbagent sync push --directory DIR --allow-delete` | `--allow-delete` removes remote configs deleted locally | +| `kbc push -d DIR --force` | `kbagent sync push --directory DIR --force` | Push's `--force` removes remote configs deleted locally (there is no `--allow-delete` flag — same flag name as pull's `--force`, but a different meaning per command) | | `kbc push --dry-run` / push-dry action | `kbagent sync push --dry-run --directory DIR` | Shows planned changes without writing | | `kbc diff -d DIR` | `kbagent sync diff --directory DIR [--json]` | `--json` gives structured drift for CI gating | | `kbc status` | `kbagent sync status --directory DIR` | | diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md b/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md index cacfa695..cb876351 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md @@ -29,21 +29,78 @@ No co-existence, but yes a controlled cutover: ## PR 1 — Conversion (the big one) → branch `migrate/kbc-to-kbagent` Do the projects one at a time; **start with a non-prod project (e.g. `dev`/`L0`)**. +**Use plain `sync init` — never `--adopt-existing`.** Root-caused on project 153 +(kbagent v0.80.0, 2026-08): `--adopt-existing` carries kbc's row paths +(`values/...` for `keboola.variables`, `codes/...` for `keboola.shared-code`) and +kbc's `relations`-based companion-config paths straight into the kbagent manifest +without translating them. That permanently leaves phantom `added`/`deleted` +entries in `sync status` (kbagent's own untracked-row scanner only recognizes a +literal `rows/` path segment; kbc's inherited `values/`/`codes/` rows never match +it), and a `sync push` against those phantom "added" rows would call +`create_config` and create duplicate sibling configs, not update the rows they +actually are. Plain `sync init` has none of this: it starts a brand-new empty +manifest and ignores every file it doesn't recognize, so pointing it at a +directory still full of kbc's `config.json`/`meta.json` tree is safe — the +following `sync pull` populates everything fresh through kbagent's own +naming/path logic, with no inherited kbc paths at all. Verified: a plain +`init`+`pull` against the same project reached `sync status` = `0 added, 0 +modified, 0 deleted` — genuinely clean, not "a small acceptable residual." + +**One required prep step: delete kbc's manifest file first.** kbc and kbagent +write to the identical path, `/.keboola/manifest.json`, and plain `sync init` +refuses to run while that file exists (`Error: Manifest already exists at +.../.keboola/manifest.json. Use 'sync pull' to update, 'sync init +--adopt-existing' ..., or delete .keboola/ to reinitialize.`) — confirmed live, +2026-08-06. This is the ONE file you delete before `init`, not the +`config.json`/`meta.json` config tree — those stay in place and get cleaned up +only after the pull below. + Per project `` (with its token in the env): ```bash export KBAGENT_PROJECT_FROM_ENV=1 KBC_TOKEN=$TOKEN \ KBC_STORAGE_API_URL=https:// -kbagent sync init --adopt-existing --project __env__ --directory -kbagent sync pull --force --project __env__ --directory # writes _config.yml +rm /.keboola/manifest.json # kbc's manifest -- same path kbagent needs +kbagent sync init --project __env__ --directory +kbagent sync pull --project __env__ --directory # writes _config.yml # Drop the orphaned kbc files kbagent does not read: find \( -name config.json -o -name meta.json \) -exec git rm -q {} + -# VERIFY BY BEHAVIOR (must be clean before you trust the project): +# VERIFY BY BEHAVIOR (must be genuinely empty before you trust the project): +kbagent sync status --directory kbagent sync diff --project __env__ --directory ``` -Acceptance for each project: `sync diff` shows `0 to create, 0 to update, 0 to delete`. -- If a handful of **scheduler / variables** configs show as "to create" (a known - wrinkle from the live test), pull again and confirm they settle; if they persist, - note them in the PR and reconcile manually before enabling push. +Acceptance for each project: `sync status` shows `0 added, 0 modified, 0 deleted` +**and** `sync diff` shows `0 to create, 0 to update, 0 to delete`. Do not accept +"a handful of leftover entries" as normal and push through it — with plain init +there should be none; if there are, stop and diagnose before moving to the next +project or enabling push. + +**Also clean up kbc-only type folders — they are left behind whole, not just +emptied of `config.json`/`meta.json`.** kbc's naming has a finer-grained +component-type taxonomy than kbagent's: kbc buckets configs into `extractor/`, +`writer/`, `transformation/`, `application/`, **`processor/`**, **`app/`** +(data apps), `_shared/` (shared code), `variables/`, `schedules/`. kbagent only +recognizes `extractor` / `writer` / `transformation` / `application` and folds +**everything else — processors, data apps, shared code — into a flat `other/`** +(`COMPONENT_TYPE_MAP` in `sync/config_format.py`; kbagent never applies kbc's +dedicated `dataAppConfig` naming template even though the manifest model still +carries the field for read compatibility). After `sync pull` rewrites those +configs under `other//...`, the old `app/`, `processor/`, and +`_shared/` directories are orphaned — but **they are not empty**: besides +`config.json`/`meta.json` (already removed above), kbc also writes +`description.md` and the code body itself (`code.sql`/`code.py`/`code.txt`/ +`code.txt` under `_shared/.../codes/...`) into these folders, none of which +kbagent reads either. A `find -empty -delete` is a no-op against them — confirmed +live, 2026-08-06 (19 leftover files, zero dirs matched `-empty`). Remove the +whole subtree instead, in the same commit as the `config.json`/`meta.json` +cleanup: +```bash +git rm -rq --ignore-unmatch /*/app /*/processor /*/_shared +``` +(adjust the glob to your branch layout — kbc nests these under the branch +directory, e.g. `main/app`, `main/processor`, `main/_shared`). Re-run `sync diff` +after — it should be unaffected (these folders were never manifest-tracked; +deleting them doesn't touch any config kbagent knows about); verified live: diff +stayed "No differences found" before and after removing all 19 leftover files. Then, still on the same branch: ```bash @@ -76,6 +133,18 @@ Merge to `main` once validate is green. - [ ] First real push: run `kbagent push` (workflow_dispatch) to `dev` first, approve, verify in the Keboola UI, then to `prod`. +**This whole sequence is an ordinary commit + ordinary PR merge — never a git +history rewrite.** The conversion diff is huge and it's tempting to reach for +`git filter-repo`, an orphan-branch reset, or a force-pushed squash of `main` to +make history "clean." Do not suggest this to a customer: it invalidates every +collaborator's clone and open PR, destroys `git blame`/audit trail across the +*entire* repo (a compliance concern, not just an inconvenience, for regulated +customers), and a coordinated force-push to `main` is exactly what most orgs' +branch-protection rules exist to block. "Close/redo open PRs, delete stale +branches, re-clone" above is already disruptive enough as ordinary git hygiene — +that is the actual answer to "the diff is too big to review," not a rewritten +history. + ## Branching model — pick one | Model | When | How | |---|---|---| @@ -86,8 +155,19 @@ For most teams already doing PR-per-change promotion, **git-branching** is the c fit and is safer (no direct prod writes from PRs). Migrate to it in PR 2, not PR 1. ## Hard guardrails (repeat to the user) -- **Never** `kbagent sync push` against an adopted-but-not-yet-pulled kbc tree — it - reports every config as "to delete" (136 in the live test) and `--allow-delete` - would wipe the project. +- **Never** `kbagent sync push` (without `--dry-run`) against an adopted-but-not-yet- + pulled kbc tree. In the original 2026-06 live test this reported every config as + "to delete" (136) and `--force` would have wiped the project; re-verified on + kbagent v0.80.0 (2026-08, same project) this no longer happens — `sync diff` now + reports untouched configs as `never_fetched`, not `deleted`. Treat the old figure as + a historical regression, not current behavior, but keep the rule: always + `sync pull` and confirm a clean `sync diff` before the first real push, on any + kbagent version. - **Never** `--allow-plaintext-on-encrypt-failure` in CI. +- **Never `--all-projects` in a migration repo directory.** It's hard-coded to + `//` for every registered project alias and (for pull) + auto-inits a fresh tree there if none exists — confirmed to silently create a + second, unrelated `/` directory alongside an existing flat manifest. + Always `--project ALIAS --directory DIR` explicit, matching what the generated + CI already does. - Keep the change freeze until `main` is converted and the first dev push is verified. diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py index 70d08f64..25ed667b 100755 --- a/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py +++ b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py @@ -232,9 +232,10 @@ def gen_push(projects: list[Project], main_branch: str, git_branching: bool) -> # the main branch, mirroring the legacy `github.ref_name == 'main'` logic. env_expr = f"${{{{ github.ref_name == '{main_branch}' && 'prod' || 'dev' }}}}" # The workflow_dispatch boolean arrives as the string 'true'/'false'; the GH - # expression maps it to the --allow-delete flag (opt-in deletion of remote - # configs that were removed locally). - delete_expr = "${{ github.event.inputs.allow_delete == 'true' && '--allow-delete' || '' }}" + # expression maps it to the --force flag (opt-in deletion of remote configs + # that were removed locally). kbagent's actual flag is --force, NOT + # --allow-delete -- there is no --allow-delete option in the CLI. + delete_expr = "${{ github.event.inputs.allow_delete == 'true' && '--force' || '' }}" steps = "".join( _project_step(p, f"push {delete_expr}", f"Push {p.directory}") for p in projects ) From ee02e2730259803c9e0d0739702e8f528bd3fd88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 6 Aug 2026 12:18:41 +0200 Subject: [PATCH 04/12] refactor(skill): trim SKILL.md bloat, clarify CI vs local auth Ponytail pass over the skill: SKILL.md had accumulated three chronological layers of investigation history (2026-06, then two 2026-08 rounds) that each superseded the last without removing it, plus a full duplicate of migration-runbook.md's per-project conversion procedure and its git-history-rewrite warning. Collapsed to a single current-state finding and pointed Step 3b at the runbook instead of re-deriving it -- same information, 383 -> 262 lines. Also clarified the Prerequisites auth bullet: the generated CI workflows always need a static per-project Storage API token (`kbagent auth login` is browser-based and can't run unattended in CI), but the local/interactive Step 3b conversion can use an already browser-authenticated + registered project alias instead of a raw token, skipping the KBAGENT_PROJECT_FROM_ENV env-injection entirely. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../skills/kbagent-cicd-migration/SKILL.md | 217 ++++-------------- 1 file changed, 48 insertions(+), 169 deletions(-) diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md index e7655e99..dad29c2b 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md @@ -50,107 +50,30 @@ hard incompatibilities make this a deliberate cutover (verified against the code by reading the reformat. Merging it to `main` is effectively a tooling version bump. - Until cutover, keep the legacy `kbc` workflows; afterwards delete them in the same PR. -> **Live-verified (project 153, GCP europe-west3, 2026-06):** -> - `KBAGENT_PROJECT_FROM_ENV=1` + `--project __env__` works for `init` and `pull`. -> - kbc pulled **143 `config.json` + 187 `meta.json`** (JSON); kbagent pulled -> **147 `_config.yml`** (YAML). Zero overlap in file format. -> - `sync init --adopt-existing` on a **kbc-produced tree** succeeds, but the very -> next `sync diff` reports **`0 to create, 0 to update, 136 to delete`** — kbagent -> does not read kbc's `config.json` at all, so it sees every existing config as a -> local deletion. **A `sync push --force` here would delete all 136 remote -> configs.** Adopt-existing adopts only the manifest, NOT the configs. -> - The correct path — adopt → `sync pull --force` (writes `_config.yml`) → `git rm` -> the orphaned `config.json`/`meta.json` — converged the diff from 136 deletes to -> ~2 (plus ~9 remote-only scheduler/variables configs that pull/diff treat -> inconsistently — verify these per project). Both file sets coexist after pull -> until you delete the kbc files, so the `git rm` step is mandatory, not optional. +> **Current finding (project 153, kbagent v0.80.0, live-verified 2026-08-06) — use +> plain `sync init`, never `--adopt-existing`, for the conversion:** +> `--adopt-existing` carries kbc's row paths (`values/{name}` for +> `keboola.variables`, `codes/{name}` for `keboola.shared-code`) straight into the +> kbagent manifest without translating them to kbagent's own `rows/{name}` +> convention. kbagent's untracked-config scanner only recognizes a literal `rows/` +> path segment, so it reports those inherited rows as brand-new top-level configs +> — confirmed live: `sync status`/`sync diff` stayed at 9 added / 1-2 deleted even +> after the orphan-file cleanup, and pushing would `create_config` duplicate +> siblings instead of updating the rows they actually are. Plain `sync init` +> avoids this: it starts a genuinely empty manifest and lets `sync pull` populate +> every path through kbagent's own naming logic from scratch — confirmed to reach +> a fully clean `sync status` ("No local changes detected") and `sync diff` ("No +> differences found") against the same project, 119 tracked configs. > -> **Re-verified (same project 153, kbagent v0.80.0, 2026-08): the 136-delete footgun -> no longer reproduces.** kbagent fixed `adopt-existing` between the original test and -> v0.80.0. Current behavior on a fresh `kbc`-produced tree (137 `config.json` + 181 -> `meta.json`): -> - `sync init --adopt-existing` + `sync diff` now reports -> `added: 0, deleted: 0, remote_only: 9, never_fetched: 110` — **no false deletes.** -> Configs kbagent hasn't pulled yet show as `never_fetched`, not `deleted`. -> - After `sync pull`, diff converges to `added: 9, deleted: 2, unchanged: 118` — a -> handful of genuine variable-row drift, not a mass delete. `sync push --dry-run` -> confirms: "would create 9, update 0, delete 2." -> - The orphaned-file claim **still holds**: 318 `config.json`/`meta.json` files -> coexisted with 146 new `_config.yml` files after pull. Deleting the orphans -> produced an *identical* `sync diff` summary before and after, confirming kbagent -> truly never reads them and the cleanup step is safe (though no longer -> safety-critical the way the delete-136 scenario was). +> One operational catch: kbc and kbagent write their manifest to the identical +> path, `.keboola/manifest.json`, so plain `sync init` refuses to run +> ("Manifest already exists...") until that one file — not the +> `config.json`/`meta.json` config tree next to it — is deleted first. Folded +> into Step 3b below. > -> **Takeaway:** re-verify the adopt/pull numbers against whichever kbagent version you -> are actually shipping — this mechanic has changed at least once. Don't quote the -> 136-delete figure as current behavior; it's a historical regression, not a standing -> hazard. The `git rm` orphan-cleanup step and the general "verify by behavior, not by -> reading the diff" guidance remain correct regardless of version. -> -> **Operator rule:** even though the mass-delete footgun is currently fixed, still -> never run `sync push` (without `--dry-run`) against an adopted-but-not-yet-pulled -> kbc tree. Always `sync pull` first, confirm `sync diff` is clean, THEN enable the -> push lane — a future regression or a customer on an older kbagent version could -> reintroduce the original failure mode. - -> **Root-caused (2026-08, project 153, kbagent v0.80.0): `--adopt-existing` never -> reaches a clean `sync status`, and DO NOT recommend it — use plain `sync init` -> instead (see "Recommended mechanic" below).** After adopt + pull, `sync status` -> still showed `added: 9, deleted: 1` even after the orphan-file cleanup. Traced to -> the manifest, not to remote drift: -> - `sync init --adopt-existing` carries kbc's row paths **verbatim** into the -> kbagent manifest — `values/{name}` for `keboola.variables`, `codes/{name}` for -> `keboola.shared-code` (kbc's own naming templates). A subsequent `sync pull` -> refreshes file *content* at an already-tracked row's existing path but never -> renames/relocates it to kbagent's own row convention. -> - kbagent's *own* row-path generator, exercised on a genuinely new row, always -> writes `rows/{name}` — confirmed by a side-by-side test: a **fresh** `sync init` -> (no adopt) + `sync pull` against the same project produced rows at -> `.../rows/default-values/_config.yml`, never `.../values/default/`. -> - `_find_untracked_configs` (the scanner behind both `sync status`'s "added" list -> and `sync push`'s create-plan) only excludes row files by checking for a literal -> `"rows"` path segment (`sync_service.py`). A row inherited at `values/...` or -> `codes/...` from adopt doesn't match that check, so **the scanner reports it as a -> brand-new top-level configuration**, with an empty `config_id` since it isn't -> actually one. **Pushing it would call `create_config` for a whole new sibling -> `keboola.variables`/`keboola.shared-code` configuration** (Phase A create path, -> not the row-update path), not update the row it actually is — a real duplicate- -> config risk on push, not just a cosmetic status mismatch. -> - Separately, kbc's own manifest.json stores **several unrelated companion -> `keboola.variables` configs at the same literal bare path `"variables"`**, -> resolving the true nesting only via a `relations` field kbagent's adopt path -> ignores entirely. Only one of those collides onto a real file after pull; the -> rest sit in the manifest with a stale `pull_hash` and no file at their recorded -> path → phantom `deleted` entries. This is a kbc-schema quirk (relation-based -> path resolution) that kbagent's manifest model has no equivalent for. -> - **A plain `sync init` (no `--adopt-existing`) + `sync pull` against the same -> project, verified twice (into an empty dir, and into a dir still containing kbc's -> untouched `config.json`/`meta.json` tree) produced `added: 0, modified: 0, -> deleted: 0, unchanged: 119` — a genuinely clean `sync status`.** Plain `init` -> ignores every foreign file it doesn't recognize; it only requires that -> `.keboola/manifest.json` not already exist. This is now the recommended -> mechanic — see below. -> -> **Re-verified end-to-end (2026-08-06, fresh `kbc init` pull of project 153 into -> an empty directory, kbc dev build + kbagent v0.80.0)**, side by side: -> - Plain `sync init` against the as-is kbc tree fails fast and by design: -> `Error: Manifest already exists at .../.keboola/manifest.json.` — because kbc -> and kbagent write to the identical path. `rm .keboola/manifest.json` (leaving -> every `config.json`/`meta.json` untouched), then plain `sync init` + `sync pull` -> reached `sync status` = "No local changes detected. (119 configurations tracked)" -> and `sync diff` = "No differences found." — genuinely, provably clean. -> - `sync init --adopt-existing` + `sync pull` against an identical untouched copy -> of the same tree reproduced the phantom-rows bug exactly as described above: -> `sync status` reported 9 added / 1 deleted, and `sync diff` planned 9 creates + -> 2 deletes — confirmed to be `keboola.variables`/`keboola.shared-code` rows -> inherited at kbc's `values/`/`codes/` paths, never relocated to kbagent's own -> `rows/` convention. -> - **Confirms the recommendation, but the write-up above was missing an -> operational step:** "plain init, no adopt-existing" cannot be run verbatim -> against a directory straight out of `kbc pull` — `.keboola/manifest.json` is -> always already there. The one-line fix, now folded into Step 3b below, is to -> `rm` that single file (never the `config.json`/`meta.json` tree it sits next -> to) immediately before the plain `init` call. +> Never run `sync push` (without `--dry-run`) against an adopted-but-not-yet- +> pulled kbc tree on any kbagent version — always `sync pull` and confirm a clean +> `sync diff` first. ### What still carries over unchanged The *orchestration shell* is CLI-agnostic: manual/scheduled pull that commits state, @@ -173,15 +96,20 @@ Four things this skill cannot infer; get them from the customer/operator first: migration (kbagent is the only tool that touches the repo from Step 3b on) — only to *verify* the "same data, new layout" claim by diffing a `kbc pull` against a `kbagent sync pull` of the same project, which is optional. -- **Storage API host + token for each project being converted.** One pair per - project (`KBC_STORAGE_API_URL` + `KBC_TOKEN`, fed via - `KBAGENT_PROJECT_FROM_ENV=1`, see "auth env vars" above) — needed for every - `sync init`/`pull`/`diff`/`push` in Step 3b. Never ask the customer to paste a - token into chat; read it via the clipboard-secret pattern or point at wherever - they already store it (a registered `kbagent project`, a CI secret, a - password manager) and reference it by path/alias, not by value. -- **Which project to convert first.** Always non-prod — confirmed explicitly in - "How to run this" below, not assumed. +- **Auth for each project being converted — two different answers for CI vs. the + local conversion step.** The generated CI workflows (Step 3/4) always need a + static per-project Storage API token secret (`KBC_TOKEN_`, + `KBAGENT_PROJECT_FROM_ENV=1`) — `kbagent auth login` is browser-based and + cannot run unattended on a GitHub Actions runner, so there is no login-based + alternative for CI. For the **local, interactive** one-time conversion in Step + 3b, though, a raw token is not the only option: if the operator already has + (or runs) `kbagent auth login` + `auth register-projects`, they get a + registered alias with a session token and can run `kbagent sync init/pull + --project --directory ` directly — no `KBAGENT_PROJECT_FROM_ENV`/ + `KBC_TOKEN` env-injection needed, since that dance exists specifically to + bridge CI's no-persisted-config environment. Either way, never ask the + customer to paste a token into chat; read it via the clipboard-secret pattern + or point at wherever they already store it and reference it by path/alias. ## How to run this — ask the customer, don't auto-pilot @@ -219,9 +147,9 @@ surface, not a batch job to execute. Concretely, stop and ask before you: If you're running this yourself against a live project to verify the skill's claims (as opposed to guiding a customer), the same discipline still applies: narrate what you're about to run and why before you run it, rather than -chaining the whole sequence unattended — that's how stale/incorrect claims in -this skill (like the historical "136 to delete" figure) go unnoticed for two -months, and how you end up cleaning up things you didn't realize you'd need to. +chaining the whole sequence unattended — that's how stale/incorrect claims go +unnoticed in a skill like this one, and how you end up cleaning up things you +didn't realize you'd need to. ## Workflow @@ -260,64 +188,15 @@ old workflows/actions in the same PR. This is the breaking part. On a fresh migration branch, for each project, convert the JSON tree to kbagent's YAML tree and remove the orphaned kbc files. -**Recommended mechanic: plain `sync init` — do NOT use `--adopt-existing`.** -`--adopt-existing` carries kbc's row paths (`values/...`, `codes/...`) and its -`relations`-based companion-config paths straight into the kbagent manifest without -translating them to kbagent's own conventions. That's confirmed to leave a -permanently-dirty `sync status` (phantom `added`/`deleted` entries — see the -root-cause note above) and, worse, makes `sync push` create duplicate sibling -configs for the misclassified rows. Plain `init` has none of this baggage: it -creates a brand-new empty manifest and ignores every file it doesn't recognize -(kbc's `config.json`/`meta.json`/legacy folders included) — **except one**: kbc -and kbagent both write their manifest to the exact same path, `.keboola/manifest.json`. -Plain `sync init` refuses to run when that file already exists (`Error: Manifest -already exists at .../.keboola/manifest.json. Use 'sync pull' to update, 'sync -init --adopt-existing' to adopt a kbc-written manifest, or delete .keboola/ to -reinitialize.`) — confirmed live, this is not a hypothetical. **You must delete -kbc's manifest file (only that one file, not the `config.json`/`meta.json` config -tree) before plain `init` will run.** Once it's gone, pointing plain `init` at a -directory that still contains kbc's `config.json`/`meta.json` tree is safe — `pull` -then populates everything through kbagent's own naming/path logic from scratch. - -```bash -git checkout -b migrate/kbc-to-kbagent -# Per project (do ONE non-prod project first and verify): -# NOTE: plain init, no --adopt-existing. kbc and kbagent share the same -# manifest path (.keboola/manifest.json), so plain init refuses to run -# until that one file is out of the way -- delete it first. -rm L0/.keboola/manifest.json -KBAGENT_PROJECT_FROM_ENV=1 KBC_TOKEN=$L0_TOKEN KBC_STORAGE_API_URL=https://connection.keboola.com \ - kbagent sync init --project __env__ --directory L0 -KBAGENT_PROJECT_FROM_ENV=1 KBC_TOKEN=$L0_TOKEN KBC_STORAGE_API_URL=https://connection.keboola.com \ - kbagent sync pull --project __env__ --directory L0 -# Remove orphaned kbc files that kbagent never reads: the config.json/meta.json -# tree, plus the kbc-only type folders (app/, processor/, _shared/) which also -# still hold description.md + code bodies kbagent never reads either -- these -# are NOT empty dirs, "find -empty" is a no-op against them (confirmed live); -# remove the whole subtree. See references/migration-runbook.md. -find L0 -name config.json -o -name meta.json | xargs git rm -q --ignore-unmatch -git rm -rq --ignore-unmatch L0/*/app L0/*/processor L0/*/_shared -git add -A -``` -Verify by **behavior**, not by reading the reformat diff: a follow-up -`sync status --directory L0` **must show 0 added, 0 modified, 0 deleted** (not just -"a small handful" — plain init reaches a genuinely empty status; if it doesn't, -something is wrong and you should stop, not push through it), and -`sync diff --project __env__ -d L0` / `sync push --dry-run` must also be clean. -Only then repeat for the remaining projects and the production lane. - -**This is a normal commit on a normal branch — never a git-history rewrite.** -It is tempting, once you see how large the reformat diff is, to reach for -`git filter-repo` / an orphan-branch reset / a force-pushed squash of `main` to -make the history "clean." **Don't. This is not something you can deliver to a -real customer:** it invalidates every collaborator's existing clone and open PR, -destroys `git blame`/audit trail across the whole repo (a compliance problem for -regulated customers, not just an inconvenience), and requires a coordinated -force-push that most orgs' branch-protection rules block outright on `main` -anyway. The migration is disruptive enough as an ordinary large commit — do not -compound it with a history rewrite. Treat the "diff is huge, don't review it -line by line, verify by behavior instead" guidance above as the actual answer to -that discomfort, not a rewritten history. +**Recommended mechanic: plain `sync init` — do NOT use `--adopt-existing`** (see +the reality-check note above for why: inherited row paths make `sync status` +permanently dirty and risk `sync push` creating duplicate configs). The exact +per-project command sequence — including the required `rm .keboola/manifest.json` +prep step, the acceptance criteria, and why it's an ordinary commit on an ordinary +branch (never a git-history rewrite, even though the reformat diff is huge) — is +maintained once in +[references/migration-runbook.md](references/migration-runbook.md) ("PR 1 — +Conversion"); follow it verbatim rather than re-deriving the steps here. ### Step 4 — Set up GitHub secrets, variables, environments The engine prints exact `gh` commands. The model: **one Storage API token secret From 6a9346081cdde952dfd5423e156d7caefd1c0964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 6 Aug 2026 13:01:45 +0200 Subject: [PATCH 05/12] fix(skill): address Copilot findings + preserve per-project directory layout - Drop the `sync init --adopt-existing ... || true` step from every generated CI run: the manifest is already committed post-conversion and checked out by actions/checkout, so init was both redundant and silently swallowing real failures (wrong token/project mismatch) via `|| true`. - Fix `--json` placement: it's a global option, not a per-subcommand flag (`kbagent --json sync diff ...`, not `sync diff --json`). - Fix wrong pull flag names in command-mapping.md (--no-storage/--no-jobs/ --with-samples, not --skip-storage/--skip-jobs/--with-table-samples). - Add the missing required --project to the sync init mapping row. - Remove the unused --git-branching flag from the generator (it never affected generation; git-branching is a per-project runtime choice made via `sync init --git-branching` + `branch-link`, documented in branching-model.md). - Fix the Step 6 "tiny diff" claim, which contradicted the reality-check's "expect a massive reformatting diff" -- clarify it's checking for drift since the conversion commit, not comparing against the original kbc tree. - De-duplicate the branching-model decision table (was in both migration-runbook.md and branching-model.md). - Explicitly document that per-project directory layout (project-id-named, L0/L1-labeled, or flat single-project) is preserved verbatim -- the generator never renames or reorganizes it, and --all-projects must not be used for exactly this reason. --- .../skills/kbagent-cicd-migration/SKILL.md | 33 +++++++++++++++---- .../references/command-mapping.md | 8 ++--- .../references/migration-runbook.md | 10 ++---- .../scripts/migrate_cicd.py | 30 ++++++++--------- 4 files changed, 48 insertions(+), 33 deletions(-) diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md index dad29c2b..3dcb6d13 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md @@ -83,6 +83,20 @@ Only three mechanics change: **install** (`uv tool install` not a binary downloa [references/command-mapping.md](references/command-mapping.md)), and **auth env vars** (`KBAGENT_PROJECT_FROM_ENV=1` + `KBC_TOKEN` + `KBC_STORAGE_API_URL`). +**The per-project directory layout is also untouched.** Whatever top-level folder +name each project already uses in the repo — a numeric project id (`9086/`), a +promotion label (`L0/`, `L1/`), or a flat single-project repo (no per-project +folder at all) — kbagent keeps it exactly as-is: `migrate_cicd.py` discovers every +project by walking for `.keboola/manifest.json` and reuses that folder's existing +path verbatim in every generated step (`--directory '{p.directory}'`); it never +renames, moves, or re-derives the folder from the alias. Inside that folder the +branch subdirectory (`main/`, etc.) and the `storage/`-adjacent component-type +folders are unchanged too — only the file format one level below (`config.json` +→ `_config.yml`) changes. **Do not** use `kbagent sync --all-projects` to "adopt" +this layout — it enforces its own `//` convention (see the +guardrail below) and would rename/duplicate the tree; the generated CI never +uses it for exactly this reason. + ## Prerequisites — ask for these before Step 1 Four things this skill cannot infer; get them from the customer/operator first: @@ -211,17 +225,22 @@ for the full mapping from the old `secrets.KBC_SAPI_TOKEN_*` / `vars.KBC_*` sche - **Branching:** the old model used a fixed branch id per env. The new model maps git branch → Keboola dev branch via `.keboola/branch-mapping.json` + `kbagent sync branch-link`. For PR-based promotion this is usually *better*: - a PR branch links to a Keboola dev branch, `main` pushes to production. Add - `--git-branching` to annotate, and walk the user through `branch-link` if they - want per-PR isolated dev branches. If they want to keep the simple single-branch - (production) model, leave branch-mapping at the default (null = production). + a PR branch links to a Keboola dev branch, `main` pushes to production. This + is a per-project runtime choice, not something the generator needs to know + about — run `kbagent sync init --git-branching` and walk the user through + `branch-link` if they want per-PR isolated dev branches (see + [references/branching-model.md](references/branching-model.md)). If they + want to keep the simple single-branch (production) model, leave + branch-mapping at the default (null = production) and skip this entirely. ### Step 6 — Validate before merging - Open the migration PR; the `kbagent-validate` workflow runs `sync diff` — confirm the diff is empty (no unintended drift) against each project. -- Manually run `kbagent-pull` once and confirm the committed state matches what - `kbc pull` produced (the layout is identical; `git diff` should be tiny — mostly - YAML vs JSON config-body formatting differences if any). +- Manually run `kbagent-pull` once **against the already-converted tree** and + confirm it's a no-op: `git diff` should be empty (or near-empty). This checks + that nothing drifted between the conversion commit and now — it is not the + same comparison as the one-time JSON→YAML conversion diff in Step 3b, which + is expected to touch every config file. - Do a `kbagent-push` dry-run (the validate workflow already does this) and read the planned changes before the first real gated push. diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md b/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md index d35d45a3..fe118238 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md @@ -15,13 +15,13 @@ installed `kbagent` version (`kbagent sync pull --help`); the new CLI evolves fa | kbc (old) | kbagent (new) | Notes | |---|---|---| -| `kbc init -d DIR --allow-target-env` | `rm DIR/.keboola/manifest.json && kbagent sync init --directory DIR` | For the one-time kbc→kbagent conversion use **plain `init`, not `--adopt-existing`** — adopting a kbc-written manifest inherits kbc's row/companion-config paths verbatim and leaves a permanently dirty `sync status` (root-caused; see SKILL.md and `migration-runbook.md`). kbc and kbagent both write to the same path (`.keboola/manifest.json`), so plain `init` errors "Manifest already exists" until you delete that one file (not the `config.json`/`meta.json` tree next to it) — confirmed live, 2026-08-06. `--adopt-existing` is still correct for re-registering an *already-converted* kbagent-native manifest in ephemeral CI (no kbc data involved at that point). | +| `kbc init -d DIR --allow-target-env` | `rm DIR/.keboola/manifest.json && kbagent sync init --project --directory DIR` | `--project` is required. For the one-time kbc→kbagent conversion use **plain `init`, not `--adopt-existing`** — adopting a kbc-written manifest inherits kbc's row/companion-config paths verbatim and leaves a permanently dirty `sync status` (root-caused; see SKILL.md and `migration-runbook.md`). kbc and kbagent both write to the same path (`.keboola/manifest.json`), so plain `init` errors "Manifest already exists" until you delete that one file (not the `config.json`/`meta.json` tree next to it) — confirmed live, 2026-08-06. `--adopt-existing` is still correct for re-registering an *already-converted* kbagent-native manifest in ephemeral CI (no kbc data involved at that point). | | `kbc persist -d DIR` | *(folded into `sync pull`)* | No separate persist step; pull writes manifest + new objects | | `kbc pull -d DIR --force` | `kbagent sync pull --directory DIR --force` | `--force` overrides local-vs-remote conflicts (3-way diff) | | `kbc push -d DIR` | `kbagent sync push --directory DIR` | Encrypts `#`-secrets fail-closed before write | | `kbc push -d DIR --force` | `kbagent sync push --directory DIR --force` | Push's `--force` removes remote configs deleted locally (there is no `--allow-delete` flag — same flag name as pull's `--force`, but a different meaning per command) | | `kbc push --dry-run` / push-dry action | `kbagent sync push --dry-run --directory DIR` | Shows planned changes without writing | -| `kbc diff -d DIR` | `kbagent sync diff --directory DIR [--json]` | `--json` gives structured drift for CI gating | +| `kbc diff -d DIR` | `kbagent [--json] sync diff --directory DIR` | `--json` is a **global** option (before `sync`, not after `diff`); gives structured drift for CI gating | | `kbc status` | `kbagent sync status --directory DIR` | | | `kbc validate` (JSON-schema) | *(no direct equivalent — gap)* | Use `sync diff` for drift; schema validation is not ported | @@ -49,8 +49,8 @@ Both CLIs honor manifest-level scoping — no command change needed: - `ignoredComponents: ["keboola.foo", ...]` — exclude component types. `kbagent` parses both (`sync/manifest.py:120`). Additionally, `sync pull` flags -`--skip-storage` / `--skip-jobs` / `--with-table-samples` control how much -*metadata* (beyond configs) is pulled — orthogonal to the config subset. +`--no-storage` / `--no-jobs` / `--with-samples` control how much *metadata* +(beyond configs) is pulled — orthogonal to the config subset. ## What has NO clean port (call out to the user) - `kbc validate` JSON-schema validation. diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md b/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md index cb876351..a0cedf1d 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md @@ -146,13 +146,9 @@ that is the actual answer to "the diff is too big to review," not a rewritten history. ## Branching model — pick one -| Model | When | How | -|---|---|---| -| **Single-branch (production)** | Each git branch/project maps straight to a production Keboola project (the demo's L0/L1 promotion) | Leave `.keboola/branch-mapping.json` at default (null = production); gate prod pushes via the GitHub Environment | -| **Git-branching (dev isolation)** | You want each PR to deploy to an isolated Keboola dev branch, then merge to prod on merge-to-main | `kbagent sync init --git-branching`; `kbagent sync branch-link --project __env__ --branch-name ` in the PR workflow; `main` maps to production | - -For most teams already doing PR-per-change promotion, **git-branching** is the closer -fit and is safer (no direct prod writes from PRs). Migrate to it in PR 2, not PR 1. +See [references/branching-model.md](branching-model.md) for the full decision table +(single-branch vs. git-branching). Migrate to git-branching in PR 2, not PR 1, if +you choose it — it's additive and doesn't require re-converting the config tree. ## Hard guardrails (repeat to the user) - **Never** `kbagent sync push` (without `--dry-run`) against an adopted-but-not-yet- diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py index 25ed667b..90b51acf 100755 --- a/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py +++ b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py @@ -22,8 +22,7 @@ Usage: python migrate_cicd.py [--write] \\ [--version 0.58.0 | --git-ref vX.Y.Z] \\ - [--main-branch main] [--schedule "0 * * * *"] \\ - [--git-branching] + [--main-branch main] [--schedule "0 * * * *"] Examples: # Inspect what would change (no writes): @@ -150,15 +149,20 @@ def _install_steps(version: str | None, git_ref: str | None) -> str: ) -def _project_step(p: Project, command: str, step_name: str) -> str: +def _project_step(p: Project, command: str, step_name: str, json_output: bool = False) -> str: """Render one per-project step using the env-injection auth model. kbagent sync is an orchestrator over registered project aliases, NOT a cwd-per-folder tool like kbc. In CI we synthesize an ephemeral project from the env (KBAGENT_PROJECT_FROM_ENV=1 -> reserved alias ``__env__``) and pass - ``--project __env__`` explicitly. A first idempotent ``init --adopt-existing`` - registers the committed manifest before the real command. + ``--project __env__`` explicitly. The committed ``.keboola/manifest.json`` + (written by the one-time conversion, Step 3b) is already checked out by + ``actions/checkout`` -- no ``sync init`` step is needed or run here; a + fresh, un-converted project has no CI step to init in the first place. + ``--json`` is a global option, so it goes before ``sync``, not after the + subcommand. """ + prefix = "kbagent --json " if json_output else "kbagent " return ( f" - name: {step_name} ({p.alias})\n" " env:\n" @@ -166,13 +170,14 @@ def _project_step(p: Project, command: str, step_name: str) -> str: f" KBC_TOKEN: ${{{{ secrets.{p.token_secret} }}}}\n" f" KBC_STORAGE_API_URL: {p.stack_url}\n" " run: |\n" - f" kbagent sync init --adopt-existing --project __env__ --directory '{p.directory}' || true\n" - f" kbagent sync {command} --project __env__ --directory '{p.directory}'\n" + f" {prefix}sync {command} --project __env__ --directory '{p.directory}'\n" ) def gen_validate(projects: list[Project], main_branch: str) -> str: - diff_steps = "".join(_project_step(p, "diff --json", f"Diff {p.directory}") for p in projects) + diff_steps = "".join( + _project_step(p, "diff", f"Diff {p.directory}", json_output=True) for p in projects + ) dry_run_steps = "".join( _project_step(p, "push --dry-run", f"Push dry-run {p.directory}") for p in projects ) @@ -227,7 +232,7 @@ def gen_pull(projects: list[Project], main_branch: str, schedule: str | None) -> ) -def gen_push(projects: list[Project], main_branch: str, git_branching: bool) -> str: +def gen_push(projects: list[Project], main_branch: str) -> str: # GitHub Environments gate production approvals; the prod environment maps to # the main branch, mirroring the legacy `github.ref_name == 'main'` logic. env_expr = f"${{{{ github.ref_name == '{main_branch}' && 'prod' || 'dev' }}}}" @@ -355,9 +360,7 @@ def run(args: argparse.Namespace) -> int: files = { ".github/workflows/kbagent-validate.yml": gen_validate(projects, args.main_branch), ".github/workflows/kbagent-pull.yml": gen_pull(projects, args.main_branch, args.schedule), - ".github/workflows/kbagent-push.yml": gen_push( - projects, args.main_branch, args.git_branching - ), + ".github/workflows/kbagent-push.yml": gen_push(projects, args.main_branch), } files = {k: v.replace(_INSTALL_TOKEN, install) for k, v in files.items()} @@ -397,9 +400,6 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument( "--schedule", default=None, help="Cron for scheduled pull, e.g. '0 * * * *' (default: none)" ) - ap.add_argument( - "--git-branching", action="store_true", help="Annotate for git-branch->Keboola-branch mode" - ) return run(ap.parse_args(argv)) From fdde1014e7dec841156ce3a07db3efd62d47ebaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 6 Aug 2026 13:14:43 +0200 Subject: [PATCH 06/12] fix(skill): fix stale line-number refs + generator edge cases from Copilot pass 2 - Replace brittle file:line citations (constants.py:425, constants.py:163, config_store.py:193, config_store.py:359) with stable symbol references -- all four had drifted from the code they were pointing at. - discover_projects now skips (with a warning) any manifest missing project.id or project.apiHost instead of silently emitting an invalid workflow (e.g. KBC_STORAGE_API_URL: https://). - _guess_repo_slug now handles repo names containing dots (e.g. "my.repo") by anchoring on end-of-line and stripping an optional .git suffix, instead of excluding "." from the repo-name character class. --- .../skills/kbagent-cicd-migration/SKILL.md | 2 +- .../references/command-mapping.md | 2 +- .../references/secrets-setup.md | 2 +- .../scripts/migrate_cicd.py | 18 +++++++++++++++--- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md index 3dcb6d13..4b1c5632 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md @@ -29,7 +29,7 @@ hard incompatibilities make this a deliberate cutover (verified against the code - `kbc` writes per config: `config.json` + `meta.json` + `description.md` (JSON). - `kbagent` writes per config: **`_config.yml`** (YAML, with `name`/`description`/ `parameters` hoisted + a `_configuration_extra` block) + extracted code files - (`constants.py:425`, `sync/config_format.py`). + (`constants.py`'s `CONFIG_FILENAME`, `sync/config_format.py`). - The first `kbagent sync pull` therefore **rewrites every configuration** into a new format. The old `config.json`/`meta.json` files are **not read** by kbagent and become orphans that must be deleted. Expect a **massive reformatting diff**. diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md b/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md index fe118238..2091c375 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md @@ -31,7 +31,7 @@ installed `kbagent` version (`kbagent sync pull --help`); the new CLI evolves fa |---|---|---| | `KBC_STORAGE_API_TOKEN` | `KBC_TOKEN` | Storage API token | | `KBC_STORAGE_API_HOST` (bare host) | `KBC_STORAGE_API_URL` (full URL) | `connection.keboola.com` → `https://connection.keboola.com` | -| *(implicit)* | `KBAGENT_PROJECT_FROM_ENV=1` | **Required** opt-in so kbagent synthesizes an ephemeral project from the env in CI (no `config.json` on disk). See `constants.py:163`, `config_store.py:193` | +| *(implicit)* | `KBAGENT_PROJECT_FROM_ENV=1` | **Required** opt-in so kbagent synthesizes an ephemeral project from the env in CI (no `config.json` on disk). See `constants.py`'s `ENV_PROJECT_FROM_ENV` and `ConfigStore._inject_env_project` | | `KBC_PROJECT_ID`, `KBC_BRANCH_ID`, `KBC_BRANCHES` | *(from manifest + branch-mapping)* | Project id comes from `.keboola/manifest.json`; branch from `.keboola/branch-mapping.json` | ## Branching diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/references/secrets-setup.md b/plugins/kbagent/skills/kbagent-cicd-migration/references/secrets-setup.md index 5b07a97f..ea2fa9c4 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/references/secrets-setup.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/references/secrets-setup.md @@ -46,7 +46,7 @@ secret model. ## Security guardrails - Do **not** commit `.kbagent/config.json` with tokens (the new CLI auto-writes a - `.gitignore` for its config dir — `config_store.py:359`). + `.gitignore` for its config dir — `ConfigStore._ensure_gitignore`). - Do **not** pass `--allow-plaintext-on-encrypt-failure` in CI. - Prefer environment-scoped secrets + required reviewers for any lane that pushes to a production project. diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py index 90b51acf..352428f4 100755 --- a/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py +++ b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py @@ -92,12 +92,20 @@ def discover_projects(repo: Path) -> list[Project]: print(f" ! skipping {manifest_path}: {exc}", file=sys.stderr) continue proj = data.get("project", {}) + project_id = proj.get("id") + api_host = proj.get("apiHost") + if not project_id or not api_host: + print( + f" ! skipping {manifest_path}: missing project.id or project.apiHost", + file=sys.stderr, + ) + continue projects.append( Project( alias=_alias_from_dir(rel), directory=rel, - project_id=str(proj.get("id", "")), - api_host=str(proj.get("apiHost", "")), + project_id=str(project_id), + api_host=str(api_host), allowed_branches=[str(b) for b in data.get("allowedBranches", [])], ignored_components=[str(c) for c in data.get("ignoredComponents", [])], ) @@ -315,7 +323,11 @@ def secrets_report(projects: list[Project], repo_slug: str) -> str: def _guess_repo_slug(repo: Path) -> str: config = repo / ".git" / "config" if config.exists(): - m = re.search(r"github\.com[:/]([^/]+/[^/\s.]+)", config.read_text(errors="ignore")) + m = re.search( + r"github\.com[:/]([^/\s]+/[^/\s]+?)(?:\.git)?\s*$", + config.read_text(errors="ignore"), + re.MULTILINE, + ) if m: return m.group(1) return "/" From d0363ca3bea1e775eb329b3e3fe4bc6f571cedd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 6 Aug 2026 13:23:00 +0200 Subject: [PATCH 07/12] fix(skill): install keboola-cli, not the legacy keboola-agent-cli name The current PyPI distribution is keboola-cli (pyproject.toml, APP_NAME_CANDIDATES in constants.py) -- keboola-agent-cli is the legacy pre-0.63 fallback name only. The generated CI, the runbook, and the command-mapping table were all pinning `uv tool install` to the legacy name, which risks installing the wrong package or failing outright once/if the legacy distribution stops being published. --- plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md | 6 +++--- .../kbagent-cicd-migration/references/command-mapping.md | 2 +- .../kbagent-cicd-migration/references/migration-runbook.md | 2 +- .../skills/kbagent-cicd-migration/scripts/migrate_cicd.py | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md index 4b1c5632..d115ac98 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md @@ -2,7 +2,7 @@ name: kbagent-cicd-migration description: > Use when migrating an existing kbc (keboola-as-code) GitHub CI/CD pipeline to - the new kbagent (keboola-agent-cli) sync engine. Covers: converting per-project + the new kbagent (keboola-cli) sync engine. Covers: converting per-project pull/push PR workflows, multi-project repos (e.g. L0/L1 dev->prod promotion), branch->environment mapping, GitHub secrets/variables/environments setup, the install step (uv tool install instead of downloading a Go binary), and the @@ -105,7 +105,7 @@ Four things this skill cannot infer; get them from the customer/operator first: and `.github/workflows/`) actually lives locally — Step 1's `migrate_cicd.py` argument. - **A `kbagent` binary or install.** Either already on `PATH`, or install it now: - `uv tool install keboola-agent-cli==` (see Step 2 for version pin) or a + `uv tool install keboola-cli==` (see Step 2 for version pin) or a downloaded standalone binary. No local `kbc` binary is required to *run* the migration (kbagent is the only tool that touches the repo from Step 3b on) — only to *verify* the "same data, new layout" claim by diffing a `kbc pull` @@ -181,7 +181,7 @@ a project" — see Step 5), and the legacy `kbc` workflow/action files it supers ### Step 2 — Pick a version pin (decide before generating) - **Pinned (recommended for prod lanes):** `--version 0.58.0` (PyPI, once published) or `--git-ref v0.58.0` (git tag, until PyPI exists). Reproducible CI. -- **Unpinned (`keboola-agent-cli`, resolves to latest):** only acceptable for a +- **Unpinned (`keboola-cli`, resolves to latest):** only acceptable for a non-prod/scratch lane. Warn the user: unpinned + the current auto-update behavior means non-deterministic CI runs. diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md b/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md index 2091c375..127d12b9 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md @@ -7,7 +7,7 @@ installed `kbagent` version (`kbagent sync pull --help`); the new CLI evolves fa | kbc (old) | kbagent (new) | |---|---| -| Download Go binary zip from `keboola/keboola-as-code` GitHub release, unzip to `/usr/local/bin/kbc` | `uv tool install keboola-agent-cli==` (PyPI) or `uv tool install 'git+https://github.com/keboola/cli@'` | +| Download Go binary zip from `keboola/keboola-as-code` GitHub release, unzip to `/usr/local/bin/kbc` | `uv tool install keboola-cli==` (PyPI) or `uv tool install 'git+https://github.com/keboola/cli@'` | | `kbc --version` | `kbagent version` | | Custom `install` composite action | `astral-sh/setup-uv@v5` + one `uv tool install` line | diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md b/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md index a0cedf1d..13f6d8e2 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md @@ -17,7 +17,7 @@ No co-existence, but yes a controlled cutover: - [ ] **Announce a change freeze** on the repo + the Keboola projects for the conversion window. Any config edit made in the UI between "pull" and "cutover" becomes drift you'll chase. Keep it short. -- [ ] **Pick a kbagent version** and pin it (`keboola-agent-cli==X.Y.Z` or +- [ ] **Pick a kbagent version** and pin it (`keboola-cli==X.Y.Z` or `git+...@vX.Y.Z`). Never unpinned on a prod lane. - [ ] **Set GitHub secrets**: one `KBC_TOKEN_` per project (see `references/secrets-setup.md`). diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py index 352428f4..c4768dc5 100755 --- a/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py +++ b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Migrate a kbc (keboola-as-code) GitHub CI/CD repo to kbagent (keboola-agent-cli). +"""Migrate a kbc (keboola-as-code) GitHub CI/CD repo to kbagent (keboola-cli). This is the engine the ``kbagent-cicd-migration`` skill drives. It: @@ -143,10 +143,10 @@ def _install_steps(version: str | None, git_ref: str | None) -> str: if git_ref: spec = f"git+https://github.com/keboola/cli@{git_ref}" elif version: - spec = f"keboola-agent-cli=={version}" + spec = f"keboola-cli=={version}" else: # Unpinned: only acceptable for non-production lanes. The skill warns. - spec = "keboola-agent-cli" + spec = "keboola-cli" return ( " - name: Install uv\n" " uses: astral-sh/setup-uv@v5\n" From b69c71a5dc52cd8f8b5768b7cb81f6797fa107d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 6 Aug 2026 13:24:19 +0200 Subject: [PATCH 08/12] fix(skill): use full relative path for project alias, not just the leaf dir _alias_from_dir took only Path(directory).name, so nested multi-project layouts (env/prod, other/prod) collided on the same KBC_TOKEN_ secret name and generated CI would push two different projects with one token. Sanitize the whole relative path instead. --- .../kbagent-cicd-migration/scripts/migrate_cicd.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py index c4768dc5..7015780a 100755 --- a/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py +++ b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py @@ -75,8 +75,15 @@ def stack_url(self) -> str: def _alias_from_dir(directory: str) -> str: - name = Path(directory).name or "PROJECT" - return re.sub(r"[^A-Za-z0-9]+", "_", name).strip("_").upper() or "PROJECT" + """Derive a secret-safe alias from the project's full relative path. + + Uses the whole path (not just the last segment) so nested multi-project + layouts (``env/prod``, ``other/prod``) don't collide on a shared + ``KBC_TOKEN_`` secret name. + """ + if directory in ("", "."): + return "PROJECT" + return re.sub(r"[^A-Za-z0-9]+", "_", directory).strip("_").upper() or "PROJECT" def discover_projects(repo: Path) -> list[Project]: From 09f6be4fc5694a27928cc97c9e2d789c92419446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 6 Aug 2026 13:33:13 +0200 Subject: [PATCH 09/12] fix(skill): drop stale 0.58.0 version examples, tighten push/diff usage-error tests - Replace the hardcoded --version 0.58.0 / v0.58.0 examples (and the "PyPI, once published" / "no PyPI yet" framing) across SKILL.md and migrate_cicd.py with an X.Y.Z placeholder -- keboola-cli has been on PyPI since well before this repo reached 0.80.0, so the old examples both pinned a stale version and implied PyPI wasn't available yet. - test_diff_without_project_is_usage_error / test_push_without_project_is_usage_error now also assert diff_all/push_all weren't called, matching the existing pull/pull_all guard test -- closes the gap where a regression routing a bad --project/--all-projects combo into the --all-projects code path could slip through undetected. --- .../kbagent/skills/kbagent-cicd-migration/SKILL.md | 7 ++++--- .../kbagent-cicd-migration/scripts/migrate_cicd.py | 14 +++++++------- tests/test_sync_cli_behavior.py | 2 ++ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md index d115ac98..1ed9dc1b 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md @@ -179,8 +179,9 @@ id / stack host / required token secret name, any `ignoredComponents` ("subset o a project" — see Step 5), and the legacy `kbc` workflow/action files it supersedes. ### Step 2 — Pick a version pin (decide before generating) -- **Pinned (recommended for prod lanes):** `--version 0.58.0` (PyPI, once published) - or `--git-ref v0.58.0` (git tag, until PyPI exists). Reproducible CI. +- **Pinned (recommended for prod lanes):** `--version X.Y.Z` (PyPI) or + `--git-ref vX.Y.Z` (git tag). Reproducible CI. Check the + [latest release](https://pypi.org/project/keboola-cli/) for the current `X.Y.Z`. - **Unpinned (`keboola-cli`, resolves to latest):** only acceptable for a non-prod/scratch lane. Warn the user: unpinned + the current auto-update behavior means non-deterministic CI runs. @@ -188,7 +189,7 @@ a project" — see Step 5), and the legacy `kbc` workflow/action files it supers ### Step 3 — Generate the clean workflows ```bash python /scripts/migrate_cicd.py /path/to/repo --write \ - --version 0.58.0 --main-branch main --schedule "0 * * * *" + --version X.Y.Z --main-branch main --schedule "0 * * * *" ``` Produces: - `.github/workflows/kbagent-validate.yml` — on PR: `sync diff` + `sync push --dry-run` per project (read-only drift + secret-encryption preflight). diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py index 7015780a..4eb1e950 100755 --- a/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py +++ b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py @@ -21,18 +21,18 @@ Usage: python migrate_cicd.py [--write] \\ - [--version 0.58.0 | --git-ref vX.Y.Z] \\ + [--version X.Y.Z | --git-ref vX.Y.Z] \\ [--main-branch main] [--schedule "0 * * * *"] Examples: # Inspect what would change (no writes): python migrate_cicd.py ../CLI-based-sync-demo - # Generate workflows pinned to a published PyPI version: - python migrate_cicd.py ../CLI-based-sync-demo --write --version 0.58.0 + # Generate workflows pinned to a PyPI version: + python migrate_cicd.py ../CLI-based-sync-demo --write --version X.Y.Z - # Pin to a git tag instead (no PyPI release yet): - python migrate_cicd.py ../CLI-based-sync-demo --write --git-ref v0.58.0 + # Pin to a git tag instead: + python migrate_cicd.py ../CLI-based-sync-demo --write --git-ref vX.Y.Z """ from __future__ import annotations @@ -409,8 +409,8 @@ def main(argv: list[str] | None = None) -> int: "--write", action="store_true", help="Write the generated workflows (default: dry-run)" ) grp = ap.add_mutually_exclusive_group() - grp.add_argument("--version", help="Pin kbagent to this PyPI version, e.g. 0.58.0") - grp.add_argument("--git-ref", help="Pin kbagent to a git tag/ref, e.g. v0.58.0 (no PyPI yet)") + grp.add_argument("--version", help="Pin kbagent to this PyPI version, e.g. X.Y.Z") + grp.add_argument("--git-ref", help="Pin kbagent to a git tag/ref, e.g. vX.Y.Z") ap.add_argument( "--main-branch", default="main", diff --git a/tests/test_sync_cli_behavior.py b/tests/test_sync_cli_behavior.py index 2beed17d..521b8f37 100644 --- a/tests/test_sync_cli_behavior.py +++ b/tests/test_sync_cli_behavior.py @@ -75,11 +75,13 @@ def test_diff_without_project_is_usage_error(self, tmp_path: Path) -> None: code, mock = _invoke(["sync", "diff", "--directory", str(tmp_path)], tmp_path) assert code == 2 mock.diff.assert_not_called() + mock.diff_all.assert_not_called() def test_push_without_project_is_usage_error(self, tmp_path: Path) -> None: code, mock = _invoke(["sync", "push", "--directory", str(tmp_path)], tmp_path) assert code == 2 mock.push.assert_not_called() + mock.push_all.assert_not_called() class TestMutuallyExclusiveSelection: From d50f558ed8272be3e6cfcb014e180a7c15de653d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 6 Aug 2026 13:41:52 +0200 Subject: [PATCH 10/12] fix(skill): drop unused main_branch params, harden test isolation - gen_validate/gen_pull no longer take an unused main_branch parameter -- only gen_push's environment-gating expression actually needs it. - Fix another stale line-number reference (sync.py:67,495) in SKILL.md, pointing at the project-selection guard by name instead. - Tests now pass an explicit --config-dir so the CLI invocation can't resolve a real on-disk config by walking up from CWD, keeping the suite hermetic regardless of the host environment it runs on. --- plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md | 3 ++- .../skills/kbagent-cicd-migration/scripts/migrate_cicd.py | 8 ++++---- tests/test_sync_cli_behavior.py | 4 +++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md index 1ed9dc1b..5b858d5e 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md @@ -35,7 +35,8 @@ hard incompatibilities make this a deliberate cutover (verified against the code and become orphans that must be deleted. Expect a **massive reformatting diff**. 2. **kbagent sync is an ORCHESTRATOR, not cwd-per-folder.** `kbc pull` runs against whatever directory you `cd` into. `kbagent sync pull` *requires* `--project ALIAS` - (resolved from a central config store) or `--all-projects` (`sync.py:67,495`). In + (resolved from a central config store) or `--all-projects` (`commands/sync.py`'s + project-selection guard). In CI we bridge this with env-injection: `KBAGENT_PROJECT_FROM_ENV=1` synthesizes a project under the reserved alias `__env__`, and every command passes `--project __env__ --directory `. diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py index 4eb1e950..dfaa92bb 100755 --- a/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py +++ b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py @@ -189,7 +189,7 @@ def _project_step(p: Project, command: str, step_name: str, json_output: bool = ) -def gen_validate(projects: list[Project], main_branch: str) -> str: +def gen_validate(projects: list[Project]) -> str: diff_steps = "".join( _project_step(p, "diff", f"Diff {p.directory}", json_output=True) for p in projects ) @@ -218,7 +218,7 @@ def gen_validate(projects: list[Project], main_branch: str) -> str: ) -def gen_pull(projects: list[Project], main_branch: str, schedule: str | None) -> str: +def gen_pull(projects: list[Project], schedule: str | None) -> str: on_block = " workflow_dispatch:\n" if schedule: on_block += f" schedule:\n - cron: '{schedule}'\n" @@ -377,8 +377,8 @@ def run(args: argparse.Namespace) -> int: install = _install_steps(args.version, args.git_ref) files = { - ".github/workflows/kbagent-validate.yml": gen_validate(projects, args.main_branch), - ".github/workflows/kbagent-pull.yml": gen_pull(projects, args.main_branch, args.schedule), + ".github/workflows/kbagent-validate.yml": gen_validate(projects), + ".github/workflows/kbagent-pull.yml": gen_pull(projects, args.schedule), ".github/workflows/kbagent-push.yml": gen_push(projects, args.main_branch), } files = {k: v.replace(_INSTALL_TOKEN, install) for k, v in files.items()} diff --git a/tests/test_sync_cli_behavior.py b/tests/test_sync_cli_behavior.py index 521b8f37..8bb2661e 100644 --- a/tests/test_sync_cli_behavior.py +++ b/tests/test_sync_cli_behavior.py @@ -58,7 +58,7 @@ def _invoke(args: list[str], tmp_path: Path) -> tuple[int, MagicMock]: MockStore.return_value = store MockProj.return_value = ProjectService(config_store=store) MockSync.return_value = mock_sync - result = runner.invoke(app, args) + result = runner.invoke(app, ["--config-dir", str(tmp_path / "config"), *args]) return result.exit_code, mock_sync @@ -131,6 +131,8 @@ def test_push_dry_run_passes_flag_and_does_not_error(self, tmp_path: Path) -> No app, [ "--json", + "--config-dir", + str(tmp_path / "config"), "sync", "push", "--project", From 2473ad29526bb6e0cc8a84ee54079782e6f639d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 6 Aug 2026 13:50:44 +0200 Subject: [PATCH 11/12] fix(skill): add required --project to command-mapping.md's pull/push/diff rows Those rows showed kbagent sync pull/push/diff with only --directory DIR, which the CLI rejects (--project ALIAS or --all-projects is required) -- copy-pasting them verbatim would fail with a usage error and contradicted the skill's own "orchestrator, not cwd-per-folder" guardrail. --- .../references/command-mapping.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md b/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md index 127d12b9..e9378301 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md @@ -17,12 +17,12 @@ installed `kbagent` version (`kbagent sync pull --help`); the new CLI evolves fa |---|---|---| | `kbc init -d DIR --allow-target-env` | `rm DIR/.keboola/manifest.json && kbagent sync init --project --directory DIR` | `--project` is required. For the one-time kbc→kbagent conversion use **plain `init`, not `--adopt-existing`** — adopting a kbc-written manifest inherits kbc's row/companion-config paths verbatim and leaves a permanently dirty `sync status` (root-caused; see SKILL.md and `migration-runbook.md`). kbc and kbagent both write to the same path (`.keboola/manifest.json`), so plain `init` errors "Manifest already exists" until you delete that one file (not the `config.json`/`meta.json` tree next to it) — confirmed live, 2026-08-06. `--adopt-existing` is still correct for re-registering an *already-converted* kbagent-native manifest in ephemeral CI (no kbc data involved at that point). | | `kbc persist -d DIR` | *(folded into `sync pull`)* | No separate persist step; pull writes manifest + new objects | -| `kbc pull -d DIR --force` | `kbagent sync pull --directory DIR --force` | `--force` overrides local-vs-remote conflicts (3-way diff) | -| `kbc push -d DIR` | `kbagent sync push --directory DIR` | Encrypts `#`-secrets fail-closed before write | -| `kbc push -d DIR --force` | `kbagent sync push --directory DIR --force` | Push's `--force` removes remote configs deleted locally (there is no `--allow-delete` flag — same flag name as pull's `--force`, but a different meaning per command) | -| `kbc push --dry-run` / push-dry action | `kbagent sync push --dry-run --directory DIR` | Shows planned changes without writing | -| `kbc diff -d DIR` | `kbagent [--json] sync diff --directory DIR` | `--json` is a **global** option (before `sync`, not after `diff`); gives structured drift for CI gating | -| `kbc status` | `kbagent sync status --directory DIR` | | +| `kbc pull -d DIR --force` | `kbagent sync pull --project --directory DIR --force` | `--project` (or `--all-projects`) is required; `--force` overrides local-vs-remote conflicts (3-way diff) | +| `kbc push -d DIR` | `kbagent sync push --project --directory DIR` | Encrypts `#`-secrets fail-closed before write | +| `kbc push -d DIR --force` | `kbagent sync push --project --directory DIR --force` | Push's `--force` removes remote configs deleted locally (there is no `--allow-delete` flag — same flag name as pull's `--force`, but a different meaning per command) | +| `kbc push --dry-run` / push-dry action | `kbagent sync push --project --dry-run --directory DIR` | Shows planned changes without writing | +| `kbc diff -d DIR` | `kbagent [--json] sync diff --project --directory DIR` | `--json` is a **global** option (before `sync`, not after `diff`); gives structured drift for CI gating | +| `kbc status` | `kbagent sync status --directory DIR` | `sync status` reads the local manifest only, no `--project` needed | | `kbc validate` (JSON-schema) | *(no direct equivalent — gap)* | Use `sync diff` for drift; schema validation is not ported | ## Auth / environment variables From 18eb782f18e8392fdaf0f6b0f00ceb389ab6f2fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Fri, 7 Aug 2026 11:14:50 +0200 Subject: [PATCH 12/12] feat(skill): recommend a PAT over a raw Storage token for CI secrets Depends on #561 (kbagent auth pat-create/pat-revoke, v0.81.0). Updates the migration skill to recommend minting a scoped Personal Access Token (kbagent auth pat-create --project-id ) for each project's KBC_TOKEN_ secret instead of pasting a raw Storage token from the Keboola UI -- a PAT is scoped to one project, has a controllable expiry (--ttl-days), and revokes independently of the account (kbagent auth pat-revoke) without touching anything else that account can do. No change to the generated GitHub Actions YAML itself: a kbc_pat_... value is a drop-in for KBC_TOKEN under KBAGENT_PROJECT_FROM_ENV=1 (kbagent detects the prefix and sends it as Authorization: Bearer automatically), so this is purely a change in how the operator obtains the secret's value. The raw Storage token path remains documented as the fallback for stacks/accounts that can't complete auth login + TOTP step-up. --- .../skills/kbagent-cicd-migration/SKILL.md | 21 +++++--- .../references/command-mapping.md | 2 +- .../references/migration-runbook.md | 5 +- .../references/secrets-setup.md | 52 +++++++++++++++++-- .../scripts/migrate_cicd.py | 7 ++- 5 files changed, 71 insertions(+), 16 deletions(-) diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md index 5b858d5e..327c347e 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/SKILL.md @@ -113,12 +113,21 @@ Four things this skill cannot infer; get them from the customer/operator first: against a `kbagent sync pull` of the same project, which is optional. - **Auth for each project being converted — two different answers for CI vs. the local conversion step.** The generated CI workflows (Step 3/4) always need a - static per-project Storage API token secret (`KBC_TOKEN_`, - `KBAGENT_PROJECT_FROM_ENV=1`) — `kbagent auth login` is browser-based and - cannot run unattended on a GitHub Actions runner, so there is no login-based - alternative for CI. For the **local, interactive** one-time conversion in Step - 3b, though, a raw token is not the only option: if the operator already has - (or runs) `kbagent auth login` + `auth register-projects`, they get a + per-project credential secret (`KBC_TOKEN_`, `KBAGENT_PROJECT_FROM_ENV=1`) + — `kbagent auth login` is browser-based and cannot run unattended on a GitHub + Actions runner, so there is no login-based alternative for CI. **Recommended + (kbagent v0.81.0+): a Personal Access Token**, minted once by the operator via + `kbagent auth login` (browser, once per stack) then `kbagent auth pat-create + --name "ci-" --project-id ` (prompts for a live TOTP code, prints + the PAT exactly once) — that PAT is a drop-in for `KBC_TOKEN`, scoped to one + project, with its own expiry and independent revocation + (`kbagent auth pat-revoke`). See + [references/secrets-setup.md](references/secrets-setup.md) for the full + walkthrough. Fall back to a raw Storage API token (pasted from the Keboola UI) + only if `auth pat-create` isn't available or the account can't complete + browser-login + TOTP. For the **local, interactive** one-time conversion in + Step 3b, a raw token is not the only option either: if the operator already + has (or runs) `kbagent auth login` + `auth register-projects`, they get a registered alias with a session token and can run `kbagent sync init/pull --project --directory ` directly — no `KBAGENT_PROJECT_FROM_ENV`/ `KBC_TOKEN` env-injection needed, since that dance exists specifically to diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md b/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md index e9378301..07dee4cd 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/references/command-mapping.md @@ -29,7 +29,7 @@ installed `kbagent` version (`kbagent sync pull --help`); the new CLI evolves fa | kbc (old) | kbagent (new) | Notes | |---|---|---| -| `KBC_STORAGE_API_TOKEN` | `KBC_TOKEN` | Storage API token | +| `KBC_STORAGE_API_TOKEN` | `KBC_TOKEN` | Recommended value: a PAT from `kbagent auth pat-create` (v0.81.0+, `kbc_pat_...`), not a raw Storage token -- kbagent detects the prefix and sends it as `Authorization: Bearer` automatically. A raw Storage token still works as a fallback. See [secrets-setup.md](secrets-setup.md) | | `KBC_STORAGE_API_HOST` (bare host) | `KBC_STORAGE_API_URL` (full URL) | `connection.keboola.com` → `https://connection.keboola.com` | | *(implicit)* | `KBAGENT_PROJECT_FROM_ENV=1` | **Required** opt-in so kbagent synthesizes an ephemeral project from the env in CI (no `config.json` on disk). See `constants.py`'s `ENV_PROJECT_FROM_ENV` and `ConfigStore._inject_env_project` | | `KBC_PROJECT_ID`, `KBC_BRANCH_ID`, `KBC_BRANCHES` | *(from manifest + branch-mapping)* | Project id comes from `.keboola/manifest.json`; branch from `.keboola/branch-mapping.json` | diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md b/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md index 13f6d8e2..4716c5a6 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/references/migration-runbook.md @@ -19,8 +19,9 @@ No co-existence, but yes a controlled cutover: becomes drift you'll chase. Keep it short. - [ ] **Pick a kbagent version** and pin it (`keboola-cli==X.Y.Z` or `git+...@vX.Y.Z`). Never unpinned on a prod lane. -- [ ] **Set GitHub secrets**: one `KBC_TOKEN_` per project (see - `references/secrets-setup.md`). +- [ ] **Set GitHub secrets**: one `KBC_TOKEN_` per project, valued with a + PAT from `kbagent auth pat-create --project-id ` (v0.81.0+), not a + raw Storage token (see `references/secrets-setup.md`). - [ ] **Create GitHub Environments** `dev` + `prod`; add required reviewers to `prod`. - [ ] **Inventory** with the skill's analyzer (dry-run): confirm every project and the legacy files it will replace. diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/references/secrets-setup.md b/plugins/kbagent/skills/kbagent-cicd-migration/references/secrets-setup.md index ea2fa9c4..8890131d 100644 --- a/plugins/kbagent/skills/kbagent-cicd-migration/references/secrets-setup.md +++ b/plugins/kbagent/skills/kbagent-cicd-migration/references/secrets-setup.md @@ -5,11 +5,46 @@ variables**, and **GitHub Environments**. The kbagent model is simpler: one toke secret per project, the stack URL baked from each manifest, environments only for push approval. +## Which credential goes in `KBC_TOKEN_`: a PAT, not a raw Storage token + +**Recommended (kbagent v0.81.0+): a Personal Access Token minted via +`kbagent auth pat-create`**, not a Storage API token copy-pasted from the UI. +The one-time setup, once per project being migrated: + +```bash +kbagent auth login --stack # once per stack; opens a browser +kbagent auth pat-create --name "ci-" --project-id +# ^ prompts for your current TOTP code, prints the PAT exactly once +``` + +Store *that* PAT as `KBC_TOKEN_` -- it is a drop-in for the env-injection +model the generated workflows already use (`KBAGENT_PROJECT_FROM_ENV=1` + +`KBC_TOKEN`): kbagent recognizes the `kbc_pat_...` prefix and sends it as +`Authorization: Bearer` automatically, so **nothing about the generated YAML +changes** -- only how the secret's value was obtained. `--project-id` scopes the +PAT to exactly that one project (least privilege for a one-secret-per-project +setup); omit it only if the CI pipeline is deliberately meant to reach every +project the signed-in user can access. Add `--read-only` for the `validate` +workflow's dry-run-only secret if you want to split read vs. write credentials +per project instead of reusing one PAT for both. + +This is *better* than the demo's raw `KBC_SAPI_TOKEN_*` model, not just a port +of it: a PAT is scoped, has an expiry you control (`--ttl-days`), and is +revocable independently of the account's password (`kbagent auth pat-revoke`) +-- rotate a compromised CI secret without touching anything else the account +can do. + +**Fallback: a raw Storage API token**, pasted from Project Settings > API +Tokens in the Keboola UI, if `auth pat-create` isn't available on your kbagent +version or your account can't complete the browser-login + TOTP step-up. This +still works exactly as before (`X-StorageApi-Token`), just without the +scoping/expiry/independent-revocation benefits above. + ## Migration table | Legacy (kbc demo) | Type | kbagent (new) | Type | Notes | |---|---|---|---|---| -| `secrets.KBC_SAPI_TOKEN_L0` | secret | `secrets.KBC_TOKEN_L0` | secret | One per project; injected as `KBC_TOKEN` | +| `secrets.KBC_SAPI_TOKEN_L0` | secret | `secrets.KBC_TOKEN_L0` | secret | One per project; injected as `KBC_TOKEN`; value is a PAT (`kbagent auth pat-create`), not a copy-pasted Storage token | | `secrets.KBC_SAPI_TOKEN_L1` | secret | `secrets.KBC_TOKEN_L1` | secret | | | `vars.KBC_SAPI_HOST` | variable | *(baked from manifest `apiHost`)* | — | Override per project in the generated `env:` block if you use a non-default stack | | `vars.KBC_PROJECT_ID_L0/L1` | variable | *(from `.keboola/manifest.json`)* | — | No longer a CI variable | @@ -21,9 +56,12 @@ push approval. ```bash REPO=/ -# One Storage API token per project (use environment-scoped secrets for prod): -gh secret set KBC_TOKEN_L0 --repo "$REPO" # paste project 9996 token -gh secret set KBC_TOKEN_L1 --repo "$REPO" # paste project 9997 token +# Mint a scoped PAT per project (see above), then store each as a secret +# (use environment-scoped secrets for prod): +kbagent auth pat-create --name "ci-L0" --project-id 9996 # copy the printed token +gh secret set KBC_TOKEN_L0 --repo "$REPO" # paste that PAT, not the account's Storage token +kbagent auth pat-create --name "ci-L1" --project-id 9997 +gh secret set KBC_TOKEN_L1 --repo "$REPO" # Environments for approval gating: gh api -X PUT "repos/$REPO/environments/dev" @@ -42,7 +80,7 @@ aliases, but that file stores tokens — unsafe to commit. In CI we instead set `KBAGENT_PROJECT_FROM_ENV=1` + `KBC_TOKEN` + `KBC_STORAGE_API_URL` per step, so the token exists only as a masked GitHub secret in the runner's env, never on disk. This is the direct, safer analog of the demo's per-project `KBC_SAPI_TOKEN_*` -secret model. +secret model -- now backed by a PAT instead of a raw copy-pasted token. ## Security guardrails - Do **not** commit `.kbagent/config.json` with tokens (the new CLI auto-writes a @@ -50,3 +88,7 @@ secret model. - Do **not** pass `--allow-plaintext-on-encrypt-failure` in CI. - Prefer environment-scoped secrets + required reviewers for any lane that pushes to a production project. +- If a `KBC_TOKEN_*` secret leaks, revoke just that PAT (`kbagent auth pat-revoke + PAT_ID`) and mint a replacement -- this is the whole reason to prefer a PAT + over a raw Storage token: revocation doesn't touch anything else the account + can do. diff --git a/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py index dfaa92bb..42dfa256 100755 --- a/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py +++ b/plugins/kbagent/skills/kbagent-cicd-migration/scripts/migrate_cicd.py @@ -300,11 +300,14 @@ def _install_steps_placeholder() -> str: def secrets_report(projects: list[Project], repo_slug: str) -> str: lines: list[str] = [] - lines.append("Required GitHub secrets (per project Storage API token):") + lines.append("Required GitHub secrets (one per project):") + lines.append(" Recommended (kbagent v0.81.0+): mint a scoped PAT, then store it --") + lines.append(" never a raw Storage token pasted from the UI:") for p in projects: + lines.append(f" kbagent auth pat-create --name 'ci-{p.alias}' --project-id {p.project_id}") lines.append( f" gh secret set {p.token_secret} " - f"--repo {repo_slug} # project {p.project_id} ({p.directory})" + f"--repo {repo_slug} # paste the PAT just printed above, for project {p.project_id} ({p.directory})" ) lines.append("") lines.append("Required GitHub Environments (for `kbagent push` approval gating):")