diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe83c05..f206c49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,6 @@ on: pull_request: branches: - main - - grok/*/* push: branches: - main @@ -16,7 +15,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - run: | exec 2>&1; set -ex tests/all.sh @@ -26,10 +25,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - name: Disable default ci-storage remote host (store locally) run: | echo -n "" > ~/ci-storage-host + - name: Create dummy files run: | echo "dummy" > dummy.txt @@ -51,6 +51,7 @@ jobs: run-before: | set -ex echo "run-before" > run-before.txt + - name: Test store (layer) uses: ./ with: @@ -58,6 +59,7 @@ jobs: storage-dir: ~/storage-dir layer-name: my-layer layer-include: layer.txt + - name: Test store (custom local-dir) uses: ./ with: @@ -73,6 +75,7 @@ jobs: action: load storage-dir: ~/storage-dir hint: aaa + - name: Check that dummy.txt and run-before.txt were restored run: | set -e @@ -88,6 +91,7 @@ jobs: action: load storage-dir: ~/storage-dir layer-name: my-layer + - name: Check that dir/subdir/layer.txt was restored, and dummy.txt still exists run: | set -e @@ -103,6 +107,7 @@ jobs: action: load storage-dir: ~/storage-dir local-dir: /tmp/dir + - name: Check that /tmp/dir/local-dir.txt was restored run: | set -e @@ -114,7 +119,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - run: | exec 2>&1; set -ex docker/ci-scaler/guest/scaler/tests/all.sh @@ -132,7 +137,7 @@ jobs: - ci-scaler-test timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - name: Start test Docker containers run: | exec 2>&1; set -ex @@ -166,7 +171,7 @@ jobs: - ci-scaler-test timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - name: Run test job inside the self-hosted runner run: echo "Hello, world!" - name: Test store using GitHub Action @@ -176,134 +181,47 @@ jobs: - name: Kill ci-runner container run: kill -SIGINT $(cat ~guest/.entrypoint.pid) - # Publishes ci-scaler image. - push-ci-scaler: + # Builds the ci-storage, ci-scaler, and ci-runner images, and on non-PR events + # pushes them to GHCR (ghcr.io//). On pull_request events the + # images are built (validating the Dockerfiles across the platform matrix) but + # not pushed, and the GHCR login is skipped. Pushes use the built-in + # GITHUB_TOKEN with packages: write, so no registry PAT is required. Decoupled + # from the self-hosted integration tests on purpose: those require CI_PAT and + # runner infra, and must not block image publishing. + push-images: runs-on: ubuntu-latest - if: github.event_name != 'pull_request' needs: - ci-storage-tool-test - ci-storage-action-test - - ci-scaler-test - - build-and-boot-containers - - spawn-job-test timeout-minutes: 15 + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + image: + - ci-storage + - ci-scaler + - ci-runner steps: - - uses: actions/checkout@v4 - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 - - uses: docker/metadata-action@v5 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + - uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 #v4.1.0 + - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 #v4.1.0 + - uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 #v6.1.0 id: meta with: - images: | - dimikot/ci-scaler - ghcr.io/${{ github.repository_owner }}/ci-scaler - - uses: docker/login-action@v3 - with: - username: dimikot - password: ${{ secrets.DOCKERHUB_PAT }} - - uses: docker/login-action@v3 + images: ghcr.io/${{ github.repository_owner }}/${{ matrix.image }} + - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee #4.2.0 + if: github.event_name != 'pull_request' with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/build-push-action@v5 + - uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf #v7.2.0 with: - context: docker/ci-scaler + context: docker/${{ matrix.image }} platforms: linux/amd64,linux/arm64,linux/arm64/v8 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - uses: peter-evans/dockerhub-description@v3 - with: - username: dimikot - password: ${{ secrets.DOCKERHUB_PAT }} - readme-filepath: docker/ci-scaler/README.md - repository: dimikot/ci-scaler - - # Publishes ci-storage image. - push-ci-storage: - runs-on: ubuntu-latest - if: github.event_name != 'pull_request' - needs: - - ci-storage-tool-test - - ci-storage-action-test - - ci-scaler-test - - build-and-boot-containers - - spawn-job-test - timeout-minutes: 15 - steps: - - uses: actions/checkout@v4 - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 - - uses: docker/metadata-action@v5 - id: meta - with: - images: | - dimikot/ci-storage - ghcr.io/${{ github.repository_owner }}/ci-storage - - uses: docker/login-action@v3 - with: - username: dimikot - password: ${{ secrets.DOCKERHUB_PAT }} - - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/build-push-action@v5 - with: - context: docker/ci-storage - platforms: linux/amd64,linux/arm64,linux/arm64/v8 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - uses: peter-evans/dockerhub-description@v3 - with: - username: dimikot - password: ${{ secrets.DOCKERHUB_PAT }} - readme-filepath: docker/ci-storage/README.md - repository: dimikot/ci-storage - - # Publishes ci-runner image. - push-ci-runner: - runs-on: ubuntu-latest - if: github.event_name != 'pull_request' - needs: - - ci-storage-tool-test - - ci-storage-action-test - - ci-scaler-test - - build-and-boot-containers - - spawn-job-test - timeout-minutes: 15 - steps: - - uses: actions/checkout@v4 - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 - - uses: docker/metadata-action@v5 - id: meta - with: - images: | - dimikot/ci-runner - ghcr.io/${{ github.repository_owner }}/ci-runner - - uses: docker/login-action@v3 - with: - username: dimikot - password: ${{ secrets.DOCKERHUB_PAT }} - - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/build-push-action@v5 - with: - context: docker/ci-runner - platforms: linux/amd64,linux/arm64,linux/arm64/v8 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - uses: peter-evans/dockerhub-description@v3 - with: - username: dimikot - password: ${{ secrets.DOCKERHUB_PAT }} - readme-filepath: docker/ci-runner/README.md - repository: dimikot/ci-runner diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..9b7a350 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,78 @@ +# CLAUDE.md + +Guidance for Claude Code when working in this repository. + +## What this repo is + +`time-loop/ci-storage` is a **mirror of the upstream `dimikot/ci-storage`** +(remote `dima`; `origin` is `time-loop/ci-storage`). It is a toolkit for running +self-hosted GitHub Actions runner infrastructure with fast work-directory +caching. It ships several things that work together: + +- **`ci-storage`** — a Bash CLI (repo root, the `ci-storage` file) that + stores/loads a work directory to/from a remote rsync-based storage host, + using `rsync --link-dest` for differential speed. Also a GitHub Action + (`action.yml`) wrapping it. +- **Three container images** built from `docker//`: + - `ci-storage` — the storage host (sshd + the CLI). + - `ci-runner` — a self-hosted runner that loads a slot from `ci-storage` on + boot, registers with GitHub, and waits for jobs. + - `ci-scaler` — scales runners from GitHub webhook signals (Python; the only + image with its own unit tests under `docker/ci-scaler/guest/scaler/tests/`). + +The primary downstream consumer of the images is a **separate repo, +`time-loop/sd`**. + +## Layout + +- `ci-storage`, `action.yml` — the CLI tool and its Action wrapper. +- `tests/` — tests for the CLI (`tests/all.sh`). +- `docker//` — one dir per image; `Dockerfile` + layered entrypoint + scripts (`root/entrypoint.NN-*.sh` run as root, `guest/entrypoint.NN-*.sh` as + the runner user, in numeric order). +- `docker/compose.yml` — local/integration testing of all three images together. +- `.github/workflows/ci.yml` — the only workflow. +- `PUBLISH.md` — release + GHCR publishing instructions. + +## CI / publishing + +`.github/workflows/ci.yml` runs on PRs to `main`, pushes to `main`, and `v*` +tags. Jobs: + +- `ci-storage-tool-test`, `ci-storage-action-test` — lightweight, always run. +- `ci-scaler-test`, `build-and-boot-containers`, `spawn-job-test` — + **self-hosted integration tests**. They require `secrets.CI_PAT` (a GitHub PAT + to register runners) and self-hosted runner infra. **These are currently red + in this org** because `CI_PAT` / the infra aren't set up yet. Treat that as a + known, separate workstream — do not assume you broke them. +- `push-images` — matrix over `[ci-storage, ci-scaler, ci-runner]`. Builds all + three; **pushes only on non-PR events** to `ghcr.io//` + (i.e. `ghcr.io/time-loop/*`). PRs build but do not push (Dockerfile + validation). Auth uses the built-in `GITHUB_TOKEN` with `packages: write` — + **no registry PAT needed**. It depends only on the two lightweight tests, so + the red integration tests never block publishing. + +### Gotchas worth knowing + +- **GHCR packages publish private by default.** Making them public is a one-time + manual UI step per package (Org → Packages → settings → change visibility); + there is **no REST API** for visibility. See `PUBLISH.md`. +- **No Docker Hub.** Upstream published to `dimikot/*` on Docker Hub; this mirror + does not (no creds, and not wanted). Don't reintroduce Docker Hub steps. +- **`ci-runner/Dockerfile` still `ADD`s the `ci-storage` tool from + `raw.githubusercontent.com/dimikot/ci-storage/main/...`.** It works (upstream + is public) but points at upstream, not this mirror — a candidate for + self-ownership later. +- The READMEs under `docker/*/` reference `ghcr.io/time-loop/*` as the canonical + pull path. `time-loop/sd` may still reference `ghcr.io/dimikot/*` until + repointed in a separate PR. + +## Conventions + +- This repo is under the `time-loop` org: commits and PR titles follow + `(): []` (lowercase subject, squad scope, + required ClickUp task id). Owning team: `@time-loop/Team-Eng-Engineering-Productivity`. +- Shell scripts: keep them shellcheck-clean (`.shellcheckrc` is configured). +- Pin third-party GitHub Actions to a commit SHA (the workflow already does + this). + diff --git a/PUBLISH.md b/PUBLISH.md index 4963779..eeeb765 100644 --- a/PUBLISH.md +++ b/PUBLISH.md @@ -1,13 +1,43 @@ # Publishing a New Version -To publish a new image to Docker Hub: +The CI workflow publishes three multi-arch container images to the GitHub +Container Registry (GHCR) under this repository's owner: + +- `ghcr.io/time-loop/ci-storage` +- `ghcr.io/time-loop/ci-scaler` +- `ghcr.io/time-loop/ci-runner` + +The `push-images` job authenticates with the built-in `GITHUB_TOKEN` +(`packages: write`), so no registry PAT is required. + +## Publish images + +Images are published on every push to `main` (tagged `main`) and on version +tags. To cut a new release: ``` git tag vA.B.C # new version git push --tags ``` -To release a new GitHub Action version to GitHub Marketplace (example for v1 +A `vA.B.C` tag produces the semver tags plus `latest`. + +## One-time: make the packages public + +New GHCR packages are created **private** by default. After the first publish, +each package must be switched to public once, via the GitHub UI (there is no +REST API to change package visibility): + +> Org → Packages → `` → Package settings → Danger Zone → +> Change visibility → Public + +Do this for `ci-storage`, `ci-scaler`, and `ci-runner`. This may require org +admin privileges. Public visibility lets downstream consumers (e.g. +`time-loop/sd`) pull the images anonymously. + +## Release a new GitHub Action version + +To release a new GitHub Action version to the GitHub Marketplace (example for v1 overwrite): ``` @@ -16,5 +46,4 @@ git tag v1 --force git push --tags --force ``` -Then open https://github.com/dimikot/ci-storage/releases/edit/v1 and click -**Update Release**. +Then open the v1 release page in the repository and click **Update Release**. diff --git a/docker/ci-runner/README.md b/docker/ci-runner/README.md index e156316..d0f1117 100644 --- a/docker/ci-runner/README.md +++ b/docker/ci-runner/README.md @@ -91,7 +91,7 @@ have a custom image on top of the default ci-runner image functionality, so it will be automatically built and started on `docker compose up`. ```Dockerfile -FROM ghcr.io/dimikot/ci-runner:latest +FROM ghcr.io/time-loop/ci-runner:latest RUN true \ && apt-get update \ && apt-get install -y nodejs redis-tools postgresql-client coreutils \ diff --git a/docker/ci-scaler/README.md b/docker/ci-scaler/README.md index 708d1f2..a2e95a3 100644 --- a/docker/ci-scaler/README.md +++ b/docker/ci-scaler/README.md @@ -32,7 +32,7 @@ Example for docker compose: services: ... ci-scaler: - image: ghcr.io/dimikot/ci-scaler:latest + image: ghcr.io/time-loop/ci-scaler:latest ports: - 18088:8088/tcp environment: diff --git a/docker/ci-storage/README.md b/docker/ci-storage/README.md index 9ce0d76..a049f9e 100644 --- a/docker/ci-storage/README.md +++ b/docker/ci-storage/README.md @@ -23,7 +23,7 @@ services: your-redis-container: ... ci-storage: - image: ghcr.io/dimikot/ci-storage:latest + image: ghcr.io/time-loop/ci-storage:latest ports: - 10022:22/tcp environment: