From 056f260fb2192982393fafc9345680a792bde2c6 Mon Sep 17 00:00:00 2001 From: Mustafa Zeydani Date: Sat, 1 Aug 2026 23:37:26 +0300 Subject: [PATCH 1/2] fix: prevent oversized homepage response headers --- .github/workflows/deploy-production.yml | 15 +++- CHANGELOG.md | 1 + README.md | 3 + devops/production/README.md | 13 ++-- devops/production/bin/release.sh | 91 +++++++++++++++++++++---- docs/deployment.md | 13 ++-- src/proxy.ts | 32 +-------- 7 files changed, 112 insertions(+), 56 deletions(-) diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 36657df..dc51fac 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -489,10 +489,11 @@ jobs: run: | health_headers="$(mktemp)" health_body="$(mktemp)" + html_headers="$(mktemp)" html_body="$(mktemp)" rsc_headers="$(mktemp)" rsc_body="$(mktemp)" - trap 'rm -f "$health_headers" "$health_body" "$html_body" "$rsc_headers" "$rsc_body"' EXIT + trap 'rm -f "$health_headers" "$health_body" "$html_headers" "$html_body" "$rsc_headers" "$rsc_body"' EXIT curl --fail --silent --show-error --retry 5 --retry-all-errors --retry-delay 5 \ -H 'Cache-Control: no-cache' -D "$health_headers" -o "$health_body" \ @@ -501,8 +502,18 @@ jobs: grep -Eiq '^cache-control:[[:space:]]*no-store([[:space:]]|,|$)' "$health_headers" curl --fail --silent --show-error --location --retry 5 --retry-all-errors --retry-delay 5 \ - -H 'Cache-Control: no-cache' -o "$html_body" https://opensyria.org/ar + -H 'Cache-Control: no-cache' -D "$html_headers" -o "$html_body" https://opensyria.org/ grep -Fq "data-dpl-id=\"${RELEASE_SHA}\"" "$html_body" + discovery_link_count="$( + awk 'BEGIN { count = 0 } { + line = tolower($0) + while (match(line, /\/\.well-known\/api-catalog/)) { + count++ + line = substr(line, RSTART + RLENGTH) + } + } END { print count }' "$html_headers" + )" + test "$discovery_link_count" -le 1 rsc_key="deploy-${RELEASE_SHA:0:12}" curl --fail --silent --show-error --retry 5 --retry-all-errors --retry-delay 5 \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 6981aea..47dcb97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Bug Fixes +* prevent duplicated discovery response headers from taking the homepage offline * include transport and telecom in the well-known API catalog from the shared discovery list * honor `[skip ci]` consistently in the website CI workflow * update Next.js and the dependency graph to patched releases and enforce a full audit in verification diff --git a/README.md b/README.md index 651de14..2d5c80e 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,9 @@ The site publishes public, read-only discovery metadata for agents: description, health endpoint, and the shared geography, universities, transport, and telecom discovery list. - `/.well-known/agent-skills/index.json` lists the available OpenSyria agent skills. +- Discovery and Markdown route responses carry the HTTP `Link` header directly. + Normal HTML responses expose the same public resources through those stable + routes without injecting a render-wide response header. - OAuth/OIDC and MCP well-known routes return explicit `404 application/problem+json` responses until OpenSyria offers protected auth flows or a public MCP server. Both `/.well-known/mcp/server-card.json` and the scanner-compatible plural alias `/.well-known/mcp/server-cards.json` use that unsupported response. ## Stack diff --git a/devops/production/README.md b/devops/production/README.md index 76e3921..e80726d 100644 --- a/devops/production/README.md +++ b/devops/production/README.md @@ -70,11 +70,14 @@ Every Docker operation uses the shared wrapper because the deployment user has no direct Docker socket access. The wrapper also preserves the temporary `DOCKER_CONFIG` across privileged pulls without persisting registry credentials. -`switch` changes the shared nginx include, validates/reloads nginx, and checks -`/health` plus `/` through `infra-nginx` with `Host: opensyria.org`. The private -check retries briefly while a graceful nginx reload drains old workers. Shared -nginx changes wait on the cross-application lock instead of failing when another -OpenSyria rollout is finishing. The previous slot is retained. +`prepare` also makes a real homepage GET against the candidate, rejects response +headers above 8 KiB, and rejects repeated agent-discovery `Link` sets before the +slot can be routed. `switch` changes the shared nginx include, +validates/reloads nginx, and checks `/health` plus a GET of `/` through +`infra-nginx` with `Host: opensyria.org`. The private check retries briefly while +a graceful nginx reload drains old workers. Shared nginx changes wait on the +cross-application lock instead of failing when another OpenSyria rollout is +finishing. The previous slot is retained. `finalize` rechecks the private route, drains existing requests, stops the previous slot, and records the new active state. diff --git a/devops/production/bin/release.sh b/devops/production/bin/release.sh index f2359a0..1f9f237 100755 --- a/devops/production/bin/release.sh +++ b/devops/production/bin/release.sh @@ -357,22 +357,83 @@ verify_direct_version() { compose exec -T "${service}" node -e ' const http = require("node:http"); const expected = process.argv[1]; - const request = http.get("http://127.0.0.1:3000/health", (response) => { - let body = ""; - response.setEncoding("utf8"); - response.on("data", (chunk) => { body += chunk; }); - response.on("end", () => { - try { - const payload = JSON.parse(body); - process.exit(response.statusCode === 200 && payload.version === expected ? 0 : 1); - } catch { - process.exit(1); - } + const publicHost = process.argv[2]; + const maxHeaderBytes = Number(process.argv[3]); + + function get(path, collectBody = false) { + return new Promise((resolve, reject) => { + const request = http.get({ + headers: { Host: publicHost }, + host: "127.0.0.1", + path, + port: 3000, + }, (response) => { + let body = ""; + if (collectBody) { + response.setEncoding("utf8"); + response.on("data", (chunk) => { body += chunk; }); + } else { + response.resume(); + } + response.on("end", () => resolve({ body, response })); + }); + request.on("error", reject); + request.setTimeout(5000, () => request.destroy(new Error(`Timed out requesting ${path}`))); }); + } + + function getRawHeaderBytes(response) { + let bytes = Buffer.byteLength( + `HTTP/${response.httpVersion} ${response.statusCode} ${response.statusMessage}\r\n` + ); + for (let index = 0; index < response.rawHeaders.length; index += 2) { + bytes += Buffer.byteLength( + `${response.rawHeaders[index]}: ${response.rawHeaders[index + 1]}\r\n` + ); + } + return bytes + 2; + } + + function countDiscoveryLinks(response) { + let count = 0; + for (let index = 0; index < response.rawHeaders.length; index += 2) { + if (response.rawHeaders[index].toLowerCase() !== "link") continue; + count += response.rawHeaders[index + 1] + .split("/.well-known/api-catalog").length - 1; + } + return count; + } + + (async () => { + const health = await get("/health", true); + const payload = JSON.parse(health.body); + if (health.response.statusCode !== 200 || payload.version !== expected) { + throw new Error(`Health endpoint did not report ${expected}`); + } + + const homepage = await get("/"); + if (homepage.response.statusCode !== 200) { + throw new Error(`Homepage returned HTTP ${homepage.response.statusCode}`); + } + + const headerBytes = getRawHeaderBytes(homepage.response); + if (headerBytes > maxHeaderBytes) { + throw new Error( + `Homepage response headers use ${headerBytes} bytes; budget is ${maxHeaderBytes}` + ); + } + + const discoveryLinks = countDiscoveryLinks(homepage.response); + if (discoveryLinks > 1) { + throw new Error( + `Homepage repeats the agent discovery Link set ${discoveryLinks} times` + ); + } + })().catch((error) => { + console.error(error.message); + process.exit(1); }); - request.on("error", () => process.exit(1)); - request.setTimeout(5000, () => request.destroy()); - ' "${expected_version}" + ' "${expected_version}" "${PUBLIC_HOST}" "8192" } verify_private_route() { @@ -390,7 +451,7 @@ verify_private_route() { )" \ && grep -Fq "\"version\":\"${expected_version}\"" <<< "${body}" \ && docker_cmd exec "${NGINX_CONTAINER}" \ - wget -q --spider \ + wget -qO /dev/null \ --header="Host: ${PUBLIC_HOST}" \ http://127.0.0.1/ 2>/dev/null; then return 0 diff --git a/docs/deployment.md b/docs/deployment.md index d3fdd61..2fd9dd3 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -46,8 +46,10 @@ Shared nginx selects the active slot through: The include contains exactly one upstream assignment. `release.sh` preserves it, writes replacements atomically, validates nginx, reloads `infra-nginx`, and -restores the prior include when a cutover check fails. Cross-application nginx -changes wait on a shared lock, and private checks retry briefly so a request that +restores the prior include when a cutover check fails. Before cutover, the +candidate must serve a homepage GET within an 8 KiB response-header budget and +must not repeat the agent-discovery `Link` set. Cross-application nginx changes +wait on a shared lock, and private GET checks retry briefly so a request that reaches a draining worker during graceful reload does not reject a healthy slot. ## GitHub Production Environment @@ -148,11 +150,12 @@ validation. activate its `current-bundle` symlink without replacing persistent state. 6. Export runtime configuration from Infisical on the host. 7. Pull the digest and prepare the inactive slot. -8. Require Docker health and an exact commit version from the slot's `/health`. +8. Require Docker health, an exact commit version from the slot's `/health`, and + a bounded, non-duplicated homepage response header block from a real GET. A failure before pending state is committed removes the attempt-owned nginx rollback backup so the next safe deployment is not blocked by an orphan. -9. Atomically switch the shared nginx include and run private Host-header smoke - checks through `infra-nginx`. +9. Atomically switch the shared nginx include and run private Host-header GET + smoke checks through `infra-nginx`. 10. Optionally verify the public health version, release marker in HTML, and an RSC response through `https://opensyria.org` after Cloudflare cutover. 11. Finalize and stop the prior slot only after every enabled check succeeds. diff --git a/src/proxy.ts b/src/proxy.ts index 6d71aa8..9427187 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -3,7 +3,6 @@ import { NextResponse } from "next/server" import createMiddleware from "next-intl/middleware" import { routing } from "./i18n/routing" -import { agentDiscoveryLinkHeader } from "./lib/agent-discovery" const intlMiddleware = createMiddleware(routing) const trackingSearchParamNames = new Set([ @@ -32,7 +31,7 @@ export default function proxy(request: NextRequest) { const cleanUrl = getCleanTrackingUrl(request) if (cleanUrl) { - return withAgentDiscoveryHeaders(NextResponse.redirect(cleanUrl, 308)) + return NextResponse.redirect(cleanUrl, 308) } if (acceptsMarkdown(request) && isMarkdownNegotiablePath(request)) { @@ -41,10 +40,10 @@ export default function proxy(request: NextRequest) { const response = NextResponse.rewrite(markdownUrl) appendHeader(response, "Vary", "Accept") - return withAgentDiscoveryHeaders(response) + return response } - return withAgentDiscoveryHeaders(intlMiddleware(request)) + return intlMiddleware(request) } function getCleanTrackingUrl(request: NextRequest) { @@ -83,31 +82,6 @@ function isMarkdownNegotiablePath(request: NextRequest) { return pathname === "/" || pathname === "/en" || pathname === "/ar" } -function withAgentDiscoveryHeaders(response: NextResponse) { - appendUniqueHeader(response, "Link", agentDiscoveryLinkHeader) - - return response -} - -function appendUniqueHeader( - response: NextResponse, - name: string, - value: string -) { - const currentValue = response.headers.get(name) - - if (!currentValue) { - response.headers.set(name, value) - return - } - - if (currentValue.includes(value)) { - return - } - - response.headers.set(name, `${currentValue}, ${value}`) -} - function appendHeader(response: NextResponse, name: string, value: string) { const currentValue = response.headers.get(name) From 557840c46841f6c23831dbe1f51f0680e9314e13 Mon Sep 17 00:00:00 2001 From: Mustafa Zeydani Date: Mon, 3 Aug 2026 04:39:51 +0300 Subject: [PATCH 2/2] fix(deploy): align production rollout with gateway --- CHANGELOG.md | 1 + devops/production/README.md | 29 ++-- devops/production/bin/release.sh | 255 ++++++++++++++----------------- docs/deployment.md | 10 +- 4 files changed, 138 insertions(+), 157 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47dcb97..de85026 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Bug Fixes +* align production rollout probes and registry authentication with the hardened Docker gateway * prevent duplicated discovery response headers from taking the homepage offline * include transport and telecom in the well-known API catalog from the shared discovery list * honor `[skip ci]` consistently in the website CI workflow diff --git a/devops/production/README.md b/devops/production/README.md index e80726d..37988eb 100644 --- a/devops/production/README.md +++ b/devops/production/README.md @@ -67,19 +67,22 @@ The host identity file is parsed with an exact key allowlist and is never executed as shell code. Every Docker operation uses the shared wrapper because the deployment user has -no direct Docker socket access. The wrapper also preserves the temporary -`DOCKER_CONFIG` across privileged pulls without persisting registry credentials. - -`prepare` also makes a real homepage GET against the candidate, rejects response -headers above 8 KiB, and rejects repeated agent-discovery `Link` sets before the -slot can be routed. `switch` changes the shared nginx include, -validates/reloads nginx, and checks `/health` plus a GET of `/` through -`infra-nginx` with `Host: opensyria.org`. The private check retries briefly while -a graceful nginx reload drains old workers. Shared nginx changes wait on the -cross-application lock instead of failing when another OpenSyria rollout is -finishing. The previous slot is retained. - -`finalize` rechecks the private route, drains existing requests, stops the +no direct Docker socket access. The release helper uses only the wrapper's typed +network probe and fixed non-secret Compose status projection. Registry +credentials live in an ephemeral private `DOCKER_CONFIG` that the wrapper copies +for the privileged image pull and the release helper removes on exit. + +`prepare` pulls and starts the candidate, then waits for its Docker healthcheck +without changing routing. `switch` changes the shared nginx include atomically, +validates/reloads nginx, and checks uncached public `/health` and homepage GETs. +The routed homepage must stay within an 8 KiB response-header budget and must +not repeat the agent-discovery `Link` set. A failed public check restores and +verifies the previous route. Checks retry briefly while a graceful nginx reload +drains old workers. Shared nginx changes wait on the cross-application lock +instead of failing when another OpenSyria rollout is finishing. The previous +slot is retained. + +`finalize` rechecks the public route, drains existing requests, stops the previous slot, and records the new active state. `rollback` restores the backed-up nginx include and previous slot. If no prior diff --git a/devops/production/bin/release.sh b/devops/production/bin/release.sh index 1f9f237..7f26b26 100755 --- a/devops/production/bin/release.sh +++ b/devops/production/bin/release.sh @@ -24,7 +24,11 @@ NGINX_DEPLOY_LOCK_FILE="/opt/syr/services/staging/.nginx-deploy.lock" NGINX_ACTIVE_INCLUDE="/opt/syr/services/staging/infrastructure/nginx/conf.d/includes/opensyria-production-website-active.conf" NGINX_CONTAINER="infra-nginx" PUBLIC_HOST="opensyria.org" +PUBLIC_URL="https://${PUBLIC_HOST}" EDGE_NETWORK="syr-staging-edge" +COMPOSE_PROJECT="opensyria-production-website" +COMPOSE_PS_FORMAT='table {{.Name}}\t{{.Image}}\t{{.State}}\t{{.Health}}' +MAX_HOMEPAGE_HEADER_BYTES="8192" HEALTH_TIMEOUT_SECONDS="${HEALTH_TIMEOUT_SECONDS:-180}" DRAIN_SECONDS="${DRAIN_SECONDS:-30}" NGINX_LOCK_TIMEOUT_SECONDS="${NGINX_LOCK_TIMEOUT_SECONDS:-300}" @@ -46,7 +50,9 @@ readonly COMPOSE_ENV_FILE RUNTIME_ENV_FILE RUNTIME_ENV_VALIDATOR readonly INFISICAL_CONFIG_FILE STATE_DIR ACTIVE_COLOR_FILE readonly ACTIVE_VERSION_FILE PENDING_FILE PREVIOUS_UPSTREAM_FILE DEPLOY_LOCK_FILE readonly NGINX_DEPLOY_LOCK_FILE NGINX_ACTIVE_INCLUDE NGINX_CONTAINER PUBLIC_HOST -readonly EDGE_NETWORK NGINX_LOCK_TIMEOUT_SECONDS NGINX_ROUTE_TIMEOUT_SECONDS +readonly PUBLIC_URL EDGE_NETWORK COMPOSE_PROJECT COMPOSE_PS_FORMAT +readonly MAX_HOMEPAGE_HEADER_BYTES NGINX_LOCK_TIMEOUT_SECONDS +readonly NGINX_ROUTE_TIMEOUT_SECONDS umask 077 @@ -314,21 +320,22 @@ reload_nginx() { && docker_cmd exec "${NGINX_CONTAINER}" nginx -s reload } -service_container_id() { - compose ps -q "$(service_for_color "$1")" +compose_status() { + compose ps --format "${COMPOSE_PS_FORMAT}" } service_is_healthy() { local color="$1" - local container_id health + local service container_prefix - container_id="$(service_container_id "${color}")" - [[ -n "${container_id}" ]] || return 1 - health="$( - docker_cmd inspect "${container_id}" \ - --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' - )" - [[ "${health}" == "healthy" ]] + service="$(service_for_color "${color}")" + container_prefix="${COMPOSE_PROJECT}-${service}-" + compose_status | awk -v prefix="${container_prefix}" ' + NR > 1 && index($1, prefix) == 1 && $3 == "running" && $4 == "healthy" { + healthy += 1 + } + END { exit(healthy == 1 ? 0 : 1) } + ' } wait_for_service_health() { @@ -340,7 +347,7 @@ wait_for_service_health() { while ! service_is_healthy "${color}"; do now="$(date +%s)" if ((now - started_at >= HEALTH_TIMEOUT_SECONDS)); then - compose ps "${service}" >&2 || true + compose_status >&2 || true compose logs --tail=120 "${service}" >&2 || true fail "Timed out waiting for ${service} to become healthy" fi @@ -348,141 +355,114 @@ wait_for_service_health() { done } -verify_direct_version() { - local color="$1" - local expected_version="$2" - local service - - service="$(service_for_color "${color}")" - compose exec -T "${service}" node -e ' - const http = require("node:http"); - const expected = process.argv[1]; - const publicHost = process.argv[2]; - const maxHeaderBytes = Number(process.argv[3]); - - function get(path, collectBody = false) { - return new Promise((resolve, reject) => { - const request = http.get({ - headers: { Host: publicHost }, - host: "127.0.0.1", - path, - port: 3000, - }, (response) => { - let body = ""; - if (collectBody) { - response.setEncoding("utf8"); - response.on("data", (chunk) => { body += chunk; }); - } else { - response.resume(); - } - response.on("end", () => resolve({ body, response })); - }); - request.on("error", reject); - request.setTimeout(5000, () => request.destroy(new Error(`Timed out requesting ${path}`))); - }); - } +probe_public_route() { + local expected_version="$1" + local enforce_header_budget="$2" + local health_body health_code homepage_headers homepage_code + local header_bytes discovery_links + + health_body="$(mktemp "${STATE_DIR}/.health-probe.XXXXXX")" + homepage_headers="$(mktemp "${STATE_DIR}/.homepage-probe.XXXXXX")" + + health_code="$( + curl -sS \ + --connect-timeout 5 \ + --max-time 10 \ + --header 'Cache-Control: no-cache' \ + --header 'Pragma: no-cache' \ + --get \ + --data-urlencode "deployment_probe=${expected_version}" \ + --output "${health_body}" \ + --write-out '%{http_code}' \ + "${PUBLIC_URL}/health" 2>/dev/null || true + )" + if [[ "${health_code}" != "200" ]] \ + || ! grep -Fq "\"version\":\"${expected_version}\"" "${health_body}"; then + rm -f -- "${health_body}" "${homepage_headers}" + return 1 + fi - function getRawHeaderBytes(response) { - let bytes = Buffer.byteLength( - `HTTP/${response.httpVersion} ${response.statusCode} ${response.statusMessage}\r\n` - ); - for (let index = 0; index < response.rawHeaders.length; index += 2) { - bytes += Buffer.byteLength( - `${response.rawHeaders[index]}: ${response.rawHeaders[index + 1]}\r\n` - ); - } - return bytes + 2; - } + homepage_code="$( + curl -sS \ + --connect-timeout 5 \ + --max-time 15 \ + --header 'Cache-Control: no-cache' \ + --header 'Pragma: no-cache' \ + --get \ + --data-urlencode "deployment_probe=${expected_version}" \ + --dump-header "${homepage_headers}" \ + --output /dev/null \ + --write-out '%{http_code}' \ + "${PUBLIC_URL}/" 2>/dev/null || true + )" + if [[ "${homepage_code}" != "200" ]]; then + rm -f -- "${health_body}" "${homepage_headers}" + return 1 + fi - function countDiscoveryLinks(response) { - let count = 0; - for (let index = 0; index < response.rawHeaders.length; index += 2) { - if (response.rawHeaders[index].toLowerCase() !== "link") continue; - count += response.rawHeaders[index + 1] - .split("/.well-known/api-catalog").length - 1; - } - return count; - } + if [[ "${enforce_header_budget}" == "true" ]]; then + header_bytes="$(wc -c < "${homepage_headers}" | tr -d '[:space:]')" + discovery_links="$( + awk ' + { + line = tolower($0) + token = "/.well-known/api-catalog" + while ((position = index(line, token)) > 0) { + count += 1 + line = substr(line, position + length(token)) + } + } + END { print count + 0 } + ' "${homepage_headers}" + )" + if ((header_bytes > MAX_HOMEPAGE_HEADER_BYTES)) \ + || ((discovery_links > 1)); then + echo "Public homepage headers use ${header_bytes} bytes and contain ${discovery_links} discovery link sets" >&2 + rm -f -- "${health_body}" "${homepage_headers}" + return 1 + fi + fi - (async () => { - const health = await get("/health", true); - const payload = JSON.parse(health.body); - if (health.response.statusCode !== 200 || payload.version !== expected) { - throw new Error(`Health endpoint did not report ${expected}`); - } - - const homepage = await get("/"); - if (homepage.response.statusCode !== 200) { - throw new Error(`Homepage returned HTTP ${homepage.response.statusCode}`); - } - - const headerBytes = getRawHeaderBytes(homepage.response); - if (headerBytes > maxHeaderBytes) { - throw new Error( - `Homepage response headers use ${headerBytes} bytes; budget is ${maxHeaderBytes}` - ); - } - - const discoveryLinks = countDiscoveryLinks(homepage.response); - if (discoveryLinks > 1) { - throw new Error( - `Homepage repeats the agent discovery Link set ${discoveryLinks} times` - ); - } - })().catch((error) => { - console.error(error.message); - process.exit(1); - }); - ' "${expected_version}" "${PUBLIC_HOST}" "8192" + rm -f -- "${health_body}" "${homepage_headers}" } -verify_private_route() { +verify_public_route() { local expected_version="$1" - local body started_at now + local enforce_header_budget="${2:-true}" + local started_at now started_at="$(date +%s)" while true; do - body="" - if body="$( - docker_cmd exec "${NGINX_CONTAINER}" \ - wget -qO- \ - --header="Host: ${PUBLIC_HOST}" \ - http://127.0.0.1/health 2>/dev/null - )" \ - && grep -Fq "\"version\":\"${expected_version}\"" <<< "${body}" \ - && docker_cmd exec "${NGINX_CONTAINER}" \ - wget -qO /dev/null \ - --header="Host: ${PUBLIC_HOST}" \ - http://127.0.0.1/ 2>/dev/null; then + if probe_public_route "${expected_version}" "${enforce_header_budget}"; then return 0 fi now="$(date +%s)" if ((now - started_at >= NGINX_ROUTE_TIMEOUT_SECONDS)); then - echo "Private ${PUBLIC_HOST} route did not stabilize on ${expected_version} within ${NGINX_ROUTE_TIMEOUT_SECONDS}s" >&2 + echo "Public ${PUBLIC_HOST} route did not stabilize on ${expected_version} within ${NGINX_ROUTE_TIMEOUT_SECONDS}s" >&2 return 1 fi sleep 1 done } -verify_previous_private_route() { +verify_previous_public_route() { [[ "${HAS_ROLLBACK}" == "true" ]] || return 1 [[ "${PREVIOUS_VERSION}" =~ ^[0-9a-f]{40}$ ]] || return 1 - verify_private_route "${PREVIOUS_VERSION}" + verify_public_route "${PREVIOUS_VERSION}" false } restore_previous_route() { if (restore_previous_upstream) \ && reload_nginx \ - && verify_previous_private_route; then + && verify_previous_public_route; then return 0 fi echo "Previous website route could not be verified; restoring the healthy candidate." >&2 if (write_nginx_upstream "${TARGET_COLOR}") \ && reload_nginx \ - && verify_private_route "${DEPLOYMENT_VERSION}"; then + && verify_public_route "${DEPLOYMENT_VERSION}"; then echo "Restored and verified the candidate website route." >&2 else echo "Candidate remains running, but automatic route recovery could not be verified." >&2 @@ -750,15 +730,19 @@ record_active_state() { write_atomic_value "${ACTIVE_VERSION_FILE}" 600 "${version}" } -login_registry() { +configure_registry() { local username="$1" local token="$2" + local encoded_auth DOCKER_CONFIG_DIR="$(mktemp -d "${ROOT_DIR}/.docker-config.XXXXXX")" chmod 700 "${DOCKER_CONFIG_DIR}" export DOCKER_CONFIG="${DOCKER_CONFIG_DIR}" - printf '%s' "${token}" \ - | docker_cmd login ghcr.io --username "${username}" --password-stdin + encoded_auth="$(printf '%s:%s' "${username}" "${token}" | base64 | tr -d '\n')" + printf '{"auths":{"ghcr.io":{"auth":"%s"}}}\n' "${encoded_auth}" \ + > "${DOCKER_CONFIG_DIR}/config.json" + chmod 600 "${DOCKER_CONFIG_DIR}/config.json" + unset encoded_auth } prepare_release() { @@ -782,9 +766,10 @@ prepare_release() { || fail "A stale or unsafe nginx backup exists without pending rollout state" IFS= read -r registry_token \ || fail "Registry token must be supplied on standard input" - [[ -n "${registry_token}" ]] || fail "Registry token is empty" + [[ -n "${registry_token}" && "${registry_token}" =~ ^[^[:space:]]+$ ]] \ + || fail "Registry token must be a single non-empty value" - docker_cmd network inspect "${EDGE_NETWORK}" >/dev/null \ + docker_cmd network-exists "${EDGE_NETWORK}" >/dev/null \ || fail "External Docker network ${EDGE_NETWORK} is missing" sync_runtime_env_from_infisical routed_color="$(current_upstream_color)" @@ -802,8 +787,7 @@ prepare_release() { "WEBSITE_${routed_color^^}_VERSION" )" if service_is_healthy "${routed_color}" \ - && verify_direct_version "${routed_color}" "${routed_version}" \ - && verify_private_route "${routed_version}"; then + && verify_public_route "${routed_version}" false; then CURRENT_COLOR="${routed_color}" HAS_ROLLBACK="true" PREVIOUS_VERSION="${routed_version}" @@ -817,19 +801,14 @@ prepare_release() { write_compose_env "${TARGET_COLOR}" "${image}" "${version}" compose config --quiet - login_registry "${registry_username}" "${registry_token}" + configure_registry "${registry_username}" "${registry_token}" registry_token="" docker_cmd pull "${image}" - docker_cmd logout ghcr.io >/dev/null 2>&1 || true target_service="$(service_for_color "${TARGET_COLOR}")" PREPARE_CLEANUP_SERVICE="${target_service}" compose up -d --no-deps --force-recreate "${target_service}" wait_for_service_health "${TARGET_COLOR}" - if ! verify_direct_version "${TARGET_COLOR}" "${version}"; then - compose logs --tail=120 "${target_service}" >&2 || true - fail "${target_service} did not report deployment version ${version}" - fi write_pending_state prepared PREPARE_CLEANUP_SERVICE="" @@ -842,14 +821,13 @@ switch_release() { [[ "${PHASE}" == "prepared" ]] || fail "Pending rollout is already switched" wait_for_service_health "${TARGET_COLOR}" - verify_direct_version "${TARGET_COLOR}" "${DEPLOYMENT_VERSION}" cmp -s "${NGINX_ACTIVE_INCLUDE}" "${PREVIOUS_UPSTREAM_FILE}" \ || fail "Shared nginx upstream changed after prepare; refusing to overwrite it" write_pending_state switching write_nginx_upstream "${TARGET_COLOR}" write_pending_state switched - if ! reload_nginx || ! verify_private_route "${DEPLOYMENT_VERSION}"; then + if ! reload_nginx || ! verify_public_route "${DEPLOYMENT_VERSION}"; then echo "Website cutover verification failed." >&2 if [[ "${HAS_ROLLBACK}" == "true" ]] && restore_previous_route; then write_pending_state prepared @@ -870,14 +848,12 @@ finalize_release() { || fail "Shared nginx is no longer routed to ${TARGET_COLOR}" wait_for_service_health "${TARGET_COLOR}" - verify_direct_version "${TARGET_COLOR}" "${DEPLOYMENT_VERSION}" - verify_private_route "${DEPLOYMENT_VERSION}" + verify_public_route "${DEPLOYMENT_VERSION}" sleep "${DRAIN_SECONDS}" [[ "$(current_upstream_color)" == "${TARGET_COLOR}" ]] \ || fail "Shared nginx changed during the drain period" wait_for_service_health "${TARGET_COLOR}" - verify_direct_version "${TARGET_COLOR}" "${DEPLOYMENT_VERSION}" - verify_private_route "${DEPLOYMENT_VERSION}" + verify_public_route "${DEPLOYMENT_VERSION}" if [[ "${HAS_ROLLBACK}" == "true" && "${CURRENT_COLOR}" != "${TARGET_COLOR}" ]]; then compose stop "$(service_for_color "${CURRENT_COLOR}")" @@ -905,10 +881,9 @@ rollback_release() { current_service="$(service_for_color "${CURRENT_COLOR}")" wait_for_service_health "${CURRENT_COLOR}" - verify_direct_version "${CURRENT_COLOR}" "${PREVIOUS_VERSION}" [[ "${PREVIOUS_VERSION}" != "${DEPLOYMENT_VERSION}" ]] \ || fail "The two colors report the same version; leaving the candidate running because the live route cannot be distinguished" - verify_previous_private_route \ + verify_previous_public_route \ || fail "The previous website route is not healthy; leaving the candidate running" compose stop "${target_service}" >/dev/null 2>&1 || true clear_pending_state @@ -923,7 +898,6 @@ rollback_release() { current_service="$(service_for_color "${CURRENT_COLOR}")" compose up -d --no-deps "${current_service}" wait_for_service_health "${CURRENT_COLOR}" - verify_direct_version "${CURRENT_COLOR}" "${PREVIOUS_VERSION}" if [[ "${routed_color}" == "${TARGET_COLOR}" ]]; then restore_previous_route \ @@ -937,7 +911,7 @@ rollback_release() { [[ "$(current_upstream_color)" == "${CURRENT_COLOR}" ]] \ || fail "Previous website route is no longer selected; the candidate was kept running" - verify_previous_private_route \ + verify_previous_public_route \ || fail "Previous website route is no longer healthy; the candidate was kept running" compose stop "${target_service}" >/dev/null 2>&1 || true record_active_state "${CURRENT_COLOR}" "${PREVIOUS_VERSION}" @@ -975,7 +949,7 @@ show_status() { if [[ -e "${COMPOSE_ENV_FILE}" || -L "${COMPOSE_ENV_FILE}" \ || -e "${RUNTIME_ENV_FILE}" || -L "${RUNTIME_ENV_FILE}" ]]; then - compose ps + compose_status fi } @@ -999,9 +973,8 @@ cleanup() { && "${routed_color}" != "${cleanup_color}" ]] \ && cmp -s "${NGINX_ACTIVE_INCLUDE}" "${PREVIOUS_UPSTREAM_FILE}" \ && service_is_healthy "${CURRENT_COLOR}" \ - && verify_direct_version "${CURRENT_COLOR}" "${PREVIOUS_VERSION}" \ && [[ "${PREVIOUS_VERSION}" != "${DEPLOYMENT_VERSION}" ]] \ - && verify_previous_private_route \ + && verify_previous_public_route \ && compose stop "${PREPARE_CLEANUP_SERVICE}" >/dev/null 2>&1 ); then : @@ -1043,6 +1016,8 @@ main() { require_real_directory "${SERVER_SERVICES_ROOT}/bin" require_real_directory "$(dirname -- "${NGINX_ACTIVE_INCLUDE}")" require_executable_regular_file "${DOCKER_WRAPPER}" + require_command base64 + require_command curl require_command flock require_command stat [[ "${HEALTH_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]{0,3}$ ]] \ diff --git a/docs/deployment.md b/docs/deployment.md index 2fd9dd3..e7c24df 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -47,10 +47,12 @@ Shared nginx selects the active slot through: The include contains exactly one upstream assignment. `release.sh` preserves it, writes replacements atomically, validates nginx, reloads `infra-nginx`, and restores the prior include when a cutover check fails. Before cutover, the -candidate must serve a homepage GET within an 8 KiB response-header budget and -must not repeat the agent-discovery `Link` set. Cross-application nginx changes -wait on a shared lock, and private GET checks retry briefly so a request that -reaches a draining worker during graceful reload does not reject a healthy slot. +candidate must pass its Docker healthcheck. After the atomic switch, uncached +public `/health` and homepage GETs must report the expected release, stay within +an 8 KiB response-header budget, and not repeat the agent-discovery `Link` set. +Cross-application nginx changes wait on a shared lock, and public checks retry +briefly so a request that reaches a draining worker during graceful reload does +not reject a healthy slot. ## GitHub Production Environment