Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ jobs:
fi
git cat-file -e "${base}^{commit}"
git cat-file -e "${HEAD_SHA}^{commit}"
./scripts/validate-image-alias-advance.sh "${base}" "${HEAD_SHA}"
garm=false
provider=false
while IFS= read -r changed; do
Expand Down
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,16 @@

## Unreleased

- Advance every image identity changed by the SQLite development headers to a
new immutable alias. Pull-request CI now rejects a manifest or provisioning
change that keeps the old alias, before a live image reconcile has to detect
the mismatch.

- Publish the existing 8192 MiB Android envelope through the release and
priority-integration GARM scale-set contracts. The private estate had already
applied those hard limits, but a later reconcile from public source would
otherwise have restored 6144 MiB.

- Publish `gha_fleet_host_root_free_percent` from each compute member's
pressure observer, excluding the Incus loop-backed thin-pool file. The
central fleet observer runs only on gha-services, so switching the ticket
Expand Down
4 changes: 2 additions & 2 deletions config/example-queue-admission.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@
"max_repository_share_percent": 75,
"capacity": {"cpu_units": 40, "memory_mib": 57344},
"scale_sets": {
"nddev-priority-integration": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 4, "reservation_memory_mib": 6144, "priority": 0},
"nddev-priority-integration": {"cpu_units": 4, "memory_mib": 8192, "reservation_cpu_units": 4, "reservation_memory_mib": 8192, "priority": 0},
"nddev-priority-standard": {"cpu_units": 2, "memory_mib": 4096, "reservation_cpu_units": 2, "reservation_memory_mib": 4096, "priority": 0},
"nddev-priority-untrusted": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 4, "reservation_memory_mib": 6144, "priority": 0},
"nddev-linux-container-canary": {"cpu_units": 2, "memory_mib": 2048, "reservation_cpu_units": 2, "reservation_memory_mib": 2048, "priority": 1},
"nddev-linux-docker-container-canary": {"cpu_units": 2, "memory_mib": 4096, "reservation_cpu_units": 2, "reservation_memory_mib": 4096, "priority": 1},
"nddev-linux-fast": {"cpu_units": 2, "memory_mib": 3072, "reservation_cpu_units": 2, "reservation_memory_mib": 3072, "priority": 1},
"nddev-linux-integration": {"cpu_units": 4, "memory_mib": 8192, "reservation_cpu_units": 4, "reservation_memory_mib": 8192, "priority": 1},
"nddev-linux-release": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 4, "reservation_memory_mib": 6144, "priority": 1},
"nddev-linux-release": {"cpu_units": 4, "memory_mib": 8192, "reservation_cpu_units": 4, "reservation_memory_mib": 8192, "priority": 1},
"nddev-linux-standard": {"cpu_units": 2, "memory_mib": 4096, "reservation_cpu_units": 2, "reservation_memory_mib": 4096, "priority": 1},
"nddev-linux-untrusted": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 4, "reservation_memory_mib": 6144, "priority": 1}
},
Expand Down
4 changes: 2 additions & 2 deletions config/example-runner-1.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,7 @@ pools:
trust: release
resources:
vcpu: 4
memory_mib: 6144
memory_mib: 8192
disk_gib: 40
max_running: 1
warm:
Expand Down Expand Up @@ -300,7 +300,7 @@ pools:
backend: linux-amd64-incus-container
scale_set_name: nddev-priority-integration
trust: trusted
resources: {vcpu: 4, memory_mib: 6144, disk_gib: 50}
resources: {vcpu: 4, memory_mib: 8192, disk_gib: 50}
max_running: 8
warm: {target_ready: 0, max_ready: 0}
capabilities: {docker: true, credentials: repository, network_policy: public-internet, cache_write_scope: trusted}
Expand Down
4 changes: 2 additions & 2 deletions config/example-runner-2.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,7 @@ pools:
trust: release
resources:
vcpu: 4
memory_mib: 6144
memory_mib: 8192
disk_gib: 40
max_running: 1
warm:
Expand Down Expand Up @@ -303,7 +303,7 @@ pools:
backend: linux-amd64-incus-container
scale_set_name: nddev-priority-integration
trust: trusted
resources: {vcpu: 4, memory_mib: 6144, disk_gib: 50}
resources: {vcpu: 4, memory_mib: 8192, disk_gib: 50}
max_running: 8
warm: {target_ready: 0, max_ready: 0}
capabilities: {docker: true, credentials: repository, network_policy: public-internet, cache_write_scope: trusted}
Expand Down
4 changes: 2 additions & 2 deletions config/example-runner-3.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ pools:
trust: release
resources:
vcpu: 4
memory_mib: 6144
memory_mib: 8192
disk_gib: 40
max_running: 1
warm:
Expand Down Expand Up @@ -305,7 +305,7 @@ pools:
backend: linux-amd64-incus-container
scale_set_name: nddev-priority-integration
trust: trusted
resources: {vcpu: 4, memory_mib: 6144, disk_gib: 50}
resources: {vcpu: 4, memory_mib: 8192, disk_gib: 50}
max_running: 8
warm: {target_ready: 0, max_ready: 0}
capabilities: {docker: true, credentials: repository, network_policy: public-internet, cache_write_scope: trusted}
Expand Down
4 changes: 2 additions & 2 deletions config/example-runner-4.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ pools:
trust: release
resources:
vcpu: 4
memory_mib: 6144
memory_mib: 8192
disk_gib: 40
max_running: 1
warm:
Expand Down Expand Up @@ -315,7 +315,7 @@ pools:
backend: linux-amd64-incus-container
scale_set_name: nddev-priority-integration
trust: trusted
resources: {vcpu: 4, memory_mib: 6144, disk_gib: 50}
resources: {vcpu: 4, memory_mib: 8192, disk_gib: 50}
max_running: 8
warm: {target_ready: 0, max_ready: 0}
capabilities: {docker: true, credentials: repository, network_policy: public-internet, cache_write_scope: trusted}
Expand Down
4 changes: 2 additions & 2 deletions config/example-services.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ pools:
trust: release
resources:
vcpu: 4
memory_mib: 6144
memory_mib: 8192
disk_gib: 40
max_running: 1
warm:
Expand Down Expand Up @@ -347,7 +347,7 @@ pools:
trust: trusted
resources:
vcpu: 4
memory_mib: 6144
memory_mib: 8192
disk_gib: 50
max_running: 8
warm:
Expand Down
2 changes: 1 addition & 1 deletion config/golden-image-container-integration.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ schema_version: 1

image:
type: container
alias: nddev-u24-amd64-ctr-docker-runner-2.336.0-r20260801-b19
alias: nddev-u24-amd64-ctr-docker-runner-2.336.0-r20260801-b20
current_alias: nddev-ubuntu-24.04-amd64-container-docker-current
previous_alias: nddev-ubuntu-24.04-amd64-container-docker-previous
source_alias: nddev-source-ubuntu-24.04-amd64-container-r20260801
Expand Down
2 changes: 1 addition & 1 deletion config/golden-image-container.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ schema_version: 1

image:
type: container
alias: nddev-ubuntu-24.04-amd64-container-runner-2.336.0-r20260801-b25
alias: nddev-ubuntu-24.04-amd64-container-runner-2.336.0-r20260801-b26
current_alias: nddev-ubuntu-24.04-amd64-container-current
previous_alias: nddev-ubuntu-24.04-amd64-container-previous
source_alias: nddev-source-ubuntu-24.04-amd64-container-r20260801
Expand Down
2 changes: 1 addition & 1 deletion config/golden-image-integration.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
schema_version: 1

image:
alias: nddev-ubuntu-24.04-amd64-docker-runner-2.336.0-r20260801-b20
alias: nddev-ubuntu-24.04-amd64-docker-runner-2.336.0-r20260801-b21
current_alias: nddev-ubuntu-24.04-amd64-docker-current
previous_alias: nddev-ubuntu-24.04-amd64-docker-previous
source_alias: nddev-source-ubuntu-24.04-amd64-r20260801
Expand Down
2 changes: 1 addition & 1 deletion config/golden-image.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
schema_version: 1

image:
alias: nddev-ubuntu-24.04-amd64-runner-2.336.0-r20260801-b21
alias: nddev-ubuntu-24.04-amd64-runner-2.336.0-r20260801-b22
current_alias: nddev-ubuntu-24.04-amd64-current
previous_alias: nddev-ubuntu-24.04-amd64-previous
source_alias: nddev-source-ubuntu-24.04-amd64-r20260801
Expand Down
19 changes: 12 additions & 7 deletions internal/garmbootstrap/fast_scale_set_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ func TestFastScaleSetResolves(t *testing.T) {
t.Fatalf("fast spec = %+v", spec)
}
// It now uses the promoted unprivileged container image; integration and
// release retain VM images as the explicit compatibility fallback.
// release use their capability-specific container images.
if spec.Image != ContainerCanaryImage {
t.Fatalf("fast image = %q, want the promoted container image", spec.Image)
}
Expand Down Expand Up @@ -52,12 +52,17 @@ func TestUntrustedScaleSetUsesDockerImageAndDistinctFlavor(t *testing.T) {
}
}

func TestReleaseScaleSetUsesStandardVMImageAndReleaseFlavor(t *testing.T) {
spec, err := resolveScaleSetSpec(ReleaseScaleSetName)
if err != nil {
t.Fatal(err)
func TestAndroidScaleSetsPublishEightGiBHardLimits(t *testing.T) {
classes := make(map[string]ScaleSetClass, len(PublishedScaleSets()))
for _, class := range PublishedScaleSets() {
classes[class.Name] = class
}
release := classes[ReleaseScaleSetName]
if release.Image != ReleaseImage || release.Flavor != ReleaseFlavor || release.MemoryMiB != 8192 {
t.Fatalf("release class=%+v", release)
}
if spec.Image != ReleaseImage || spec.Flavor != ReleaseFlavor {
t.Fatalf("release spec=%+v", spec)
priority := classes[PriorityIntegrationScaleSetName]
if priority.Image != PriorityIntegrationImage || priority.Flavor != PriorityIntegrationFlavor || priority.MemoryMiB != 8192 {
t.Fatalf("priority integration class=%+v", priority)
}
}
14 changes: 7 additions & 7 deletions internal/garmbootstrap/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,17 +36,17 @@ const (
DefaultPoolBalancerType = "roundrobin"
DefaultProviderName = "nddev-incus"
DefaultImage = "nddev-ubuntu-24.04-amd64-container-current"
IntegrationImage = "nddev-u24-amd64-ctr-docker-runner-2.336.0-r20260801-b19"
PriorityStandardImage = "nddev-ubuntu-24.04-amd64-container-runner-2.336.0-r20260801-b25"
PriorityIntegrationImage = "nddev-u24-amd64-ctr-docker-runner-2.336.0-r20260801-b19"
IntegrationImage = "nddev-u24-amd64-ctr-docker-runner-2.336.0-r20260801-b20"
PriorityStandardImage = "nddev-ubuntu-24.04-amd64-container-runner-2.336.0-r20260801-b26"
PriorityIntegrationImage = "nddev-u24-amd64-ctr-docker-runner-2.336.0-r20260801-b20"
// Every Linux class is an ephemeral Incus container. Docker-capable classes
// use their nested-runtime image; release uses a separately stage-smoked
// standard image so OIDC authority does not inherit Docker/nesting.
FastImage = ContainerCanaryImage
UntrustedImage = IntegrationImage
ReleaseImage = "nddev-ubuntu-24.04-amd64-container-runner-2.336.0-r20260801-b25"
ReleaseImage = "nddev-ubuntu-24.04-amd64-container-runner-2.336.0-r20260801-b26"
ContainerCanaryImage = "nddev-ubuntu-24.04-amd64-container-current"
DockerContainerCanaryImage = "nddev-u24-amd64-ctr-docker-runner-2.336.0-r20260801-b19"
DockerContainerCanaryImage = "nddev-u24-amd64-ctr-docker-runner-2.336.0-r20260801-b20"
DefaultFlavor = "nddev-linux-standard"
IntegrationFlavor = "nddev-linux-integration"
FastFlavor = "nddev-linux-fast"
Expand Down Expand Up @@ -104,11 +104,11 @@ func PublishedScaleSets() []ScaleSetClass {
class(IntegrationScaleSetName, IntegrationImage, IntegrationFlavor, 8, "trusted", "repository", "public-internet", "trusted", true, true, 4, 8192, 50, false),
class(FastScaleSetName, FastImage, FastFlavor, 16, "trusted", "repository", "public-internet", "trusted", false, false, 2, 3072, 30, false),
class(UntrustedScaleSetName, UntrustedImage, UntrustedFlavor, 8, "untrusted", "none", "public-internet", "none", true, false, 4, 6144, 50, false),
class(ReleaseScaleSetName, ReleaseImage, ReleaseFlavor, 1, "release", "oidc-only", "release-allowlist", "none", false, false, 4, 6144, 40, false),
class(ReleaseScaleSetName, ReleaseImage, ReleaseFlavor, 1, "release", "oidc-only", "release-allowlist", "none", false, false, 4, 8192, 40, false),
class(ContainerCanaryScaleSetName, ContainerCanaryImage, ContainerCanaryFlavor, 12, "trusted", "none", "public-internet", "none", false, false, 2, 2048, 30, false),
class(DockerContainerCanaryScaleSetName, DockerContainerCanaryImage, DockerContainerCanaryFlavor, 1, "trusted", "repository", "public-internet", "none", true, false, 2, 4096, 40, false),
class(PriorityStandardScaleSetName, PriorityStandardImage, PriorityStandardFlavor, 16, "trusted", "repository", "public-internet", "trusted", false, false, 2, 4096, 30, true),
class(PriorityIntegrationScaleSetName, PriorityIntegrationImage, PriorityIntegrationFlavor, 8, "trusted", "repository", "public-internet", "trusted", true, true, 4, 6144, 50, true),
class(PriorityIntegrationScaleSetName, PriorityIntegrationImage, PriorityIntegrationFlavor, 8, "trusted", "repository", "public-internet", "trusted", true, true, 4, 8192, 50, true),
class(PriorityUntrustedScaleSetName, UntrustedImage, PriorityUntrustedFlavor, 8, "untrusted", "none", "public-internet", "none", true, false, 4, 6144, 50, true),
}
}
Expand Down
2 changes: 1 addition & 1 deletion internal/hostcapacity/capacity_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ func TestDeclaredCapsAlreadyImplyTheCeiling(t *testing.T) {
{Pool: "nddev-linux-standard", Workers: 3, Binding: BindingMemory},
{Pool: "nddev-linux-integration", Workers: 1, Binding: BindingMemory},
{Pool: "nddev-linux-untrusted", Workers: 2, Binding: BindingCPU},
{Pool: "nddev-linux-release", Workers: 2, Binding: BindingCPU},
{Pool: "nddev-linux-release", Workers: 1, Binding: BindingMemory},
} {
got, err := ForPool(cfg, expected.Pool)
if err != nil {
Expand Down
17 changes: 6 additions & 11 deletions internal/imagebuild/orchestrator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,17 +155,12 @@ func TestRecipeFingerprintIsDeterministic(t *testing.T) {
if first != second || !strings.HasPrefix(first, "sha256:") || len(first) != 71 {
t.Fatalf("unexpected recipe fingerprints %q %q", first, second)
}
// Pinned so a recipe change has to be stated rather than noticed. It has
// moved with every recipe change -- the b15 alias, the restored warm agent,
// the runner-owned /home/runner/.cache -- and moves here because b22 pins
// the runner's uid and gid, skips the warm-up on a one-job worker, lets
// cloud-init exit when there is no cloud-config to apply, and masks the
// server units a one-job worker never needs. b21 is built and promoted, so
// its contents cannot change under it -- the alias goes to b22. That
// coupling is the point: the alias is part of the recipe, so a manifest
// whose contents changed under an unchanged alias would otherwise ask the
// builder to produce different bytes for a name that is already promoted.
if first != "sha256:7a3a1820816163ead5bbbfecfd455147adcae39195b8d6d194927408f7e2c40b" {
// Pinned so a recipe change has to be stated rather than noticed. The
// manifest now guarantees the SQLite development headers required by
// release workers. The prior image is already promoted, so the alias moves
// with the recipe instead of asking the builder to produce different bytes
// for an immutable name.
if first != "sha256:c8c7dd85141e4958052a580251ee1b9fd66f71e2fdee25205853c456ff120c1c" {
t.Fatalf("deployed standard recipe fingerprint drifted: %q", first)
}
smoke, err := SmokeFingerprint(plan)
Expand Down
2 changes: 1 addition & 1 deletion internal/imagemanifest/manifest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func TestRepositoryManifestIsValidAndPinned(t *testing.T) {
if !strings.HasPrefix(fingerprint, "sha256:") || len(fingerprint) != len("sha256:")+64 {
t.Fatalf("unexpected fingerprint %q", fingerprint)
}
if fingerprint != "sha256:98ab78a52fdccdd6df1622e5383f27a67c507cfdad789880c0cf10b1b7f31377" {
if fingerprint != "sha256:ecb5434b09d2aa042b5d400115974741f733c2950f1a32af56c792c7655158ce" {
t.Fatalf("standard manifest fingerprint drifted: %q", fingerprint)
}
}
Expand Down
61 changes: 61 additions & 0 deletions scripts/validate-image-alias-advance.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
#!/usr/bin/env bash
set -euo pipefail

base_ref="${1:?base ref is required}"
head_ref="${2:-HEAD}"

manifests=(
config/golden-image.yaml
config/golden-image-integration.yaml
config/golden-image-container.yaml
config/golden-image-container-integration.yaml
)

alias_at() {
git show "$1:$2" | awk '$1 == "alias:" { print $2; exit }'
}

manifest_payload_at() {
git show "$1:$2" |
awk '
/^[[:space:]]*($|#)/ { next }
/^[[:space:]]*alias:[[:space:]]/ { next }
{ sub(/[[:space:]]+#.*$/, ""); print }
'
}

changed_payload() {
! cmp -s \
<(manifest_payload_at "${base_ref}" "$1") \
<(manifest_payload_at "${head_ref}" "$1")
}

require_alias_advance() {
local manifest="$1"
local reason="$2"
local before after
before="$(alias_at "${base_ref}" "${manifest}")"
after="$(alias_at "${head_ref}" "${manifest}")"
if [[ -z "${before}" || -z "${after}" || "${before}" == "${after}" ]]; then
printf '%s changed image bytes for %s without advancing its immutable alias (%s)\n' \
"${reason}" "${manifest}" "${before:-missing}" >&2
return 1
fi
}

for manifest in "${manifests[@]}"; do
if changed_payload "${manifest}"; then
require_alias_advance "${manifest}" "manifest payload"
fi
done

if ! git diff --quiet "${base_ref}" "${head_ref}" -- internal/imagebuild/assets/provision.sh; then
for manifest in "${manifests[@]}"; do
require_alias_advance "${manifest}" "shared provisioning"
done
fi

if ! git diff --quiet "${base_ref}" "${head_ref}" -- internal/imagebuild/assets/docker-provision.sh; then
require_alias_advance config/golden-image-integration.yaml "Docker provisioning"
require_alias_advance config/golden-image-container-integration.yaml "Docker provisioning"
fi