From 86735d032cf5877fe51c693406b4d32f11ad298f Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Tue, 25 Aug 2026 23:18:57 -0700 Subject: [PATCH] Fix Kubernetes control-plane restart after IP rotation --- Sources/ContainerK8s/Commands/K8sCreate.swift | 4 ++ Sources/ContainerK8s/K8sHelper.swift | 2 + .../Support/K8sHelper+Bootstrap.swift | 14 ++++-- .../K8s/TestK8sRunSerial.swift | 49 +++++++++++++++++++ Tests/K8sPluginTests/K8sBootstrapTests.swift | 41 ++++++++++++++++ 5 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 Tests/K8sPluginTests/K8sBootstrapTests.swift diff --git a/Sources/ContainerK8s/Commands/K8sCreate.swift b/Sources/ContainerK8s/Commands/K8sCreate.swift index 3f5d44155..7970bbc71 100644 --- a/Sources/ContainerK8s/Commands/K8sCreate.swift +++ b/Sources/ContainerK8s/Commands/K8sCreate.swift @@ -102,6 +102,10 @@ public struct K8sCreate: AsyncParsableCommand { try await K8sHelper.prepareNode(nodeID: name, client: client, log: log) try await K8sHelper.bootstrapControlPlane( nodeID: name, apiServerSANs: sans, advertiseAddress: vmIP, + // All in-VM Kubernetes clients (including host-networked kube-proxy) + // can reach the single-node API through loopback. This endpoint is + // independent of the container's rotating vmnet address. + controlPlaneEndpoint: K8sHelper.nodeLocalControlPlaneEndpoint, schedulable: provisioner.roles.contains(StandardRoles.worker), client: client, log: log) diff --git a/Sources/ContainerK8s/K8sHelper.swift b/Sources/ContainerK8s/K8sHelper.swift index 2bc4c9d7f..2fa10dfb9 100644 --- a/Sources/ContainerK8s/K8sHelper.swift +++ b/Sources/ContainerK8s/K8sHelper.swift @@ -51,6 +51,8 @@ public struct K8sHelper { static let proxyEnvVars = ["HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"] public static let clusterContainerPort: UInt16 = 6443 + /// Endpoint used by clients inside the single-node cluster container. + static let nodeLocalControlPlaneEndpoint = "127.0.0.1:\(clusterContainerPort)" // MARK: - Resource defaults diff --git a/Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift b/Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift index 964f9b72f..ca203e11e 100644 --- a/Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift +++ b/Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift @@ -35,12 +35,15 @@ extension K8sHelper { static func bootstrapControlPlane( nodeID: String, apiServerSANs: [String], advertiseAddress: String, + controlPlaneEndpoint: String, schedulable: Bool, client: ContainerClient, log: Logger ) async throws { - let configYAML = initConfigYAML(advertiseAddress: advertiseAddress, certSANs: apiServerSANs) + let configYAML = initConfigYAML( + advertiseAddress: advertiseAddress, certSANs: apiServerSANs, + controlPlaneEndpoint: controlPlaneEndpoint) var r = try await execCapture( containerId: nodeID, executable: "/bin/sh", - arguments: ["-c", "cat > /etc/kubernetes/kubeadm-config.yaml <<'EOF'\n\(configYAML)\nEOF"], + arguments: ["-c", "mkdir -p /kind && cat > /kind/kubeadm.conf <<'EOF'\n\(configYAML)\nEOF"], client: client) guard r.code == 0 else { throw ContainerizationError(.internalError, message: "write kubeadm config failed on \(nodeID): \(r.output)") @@ -50,7 +53,7 @@ extension K8sHelper { r = try await execCapture( containerId: nodeID, executable: kubeadmPath, arguments: [ - "init", "--config", "/etc/kubernetes/kubeadm-config.yaml", + "init", "--config", "/kind/kubeadm.conf", "--ignore-preflight-errors", ignorePreflightErrors, ], client: client) @@ -135,7 +138,9 @@ extension K8sHelper { """ } - private static func initConfigYAML(advertiseAddress: String, certSANs: [String]) -> String { + static func initConfigYAML( + advertiseAddress: String, certSANs: [String], controlPlaneEndpoint: String + ) -> String { let sans = certSANs.map { " - \($0)" }.joined(separator: "\n") return """ apiVersion: kubeadm.k8s.io/v1beta4 @@ -148,6 +153,7 @@ extension K8sHelper { --- apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration + controlPlaneEndpoint: \(controlPlaneEndpoint) kubernetesVersion: \(kubernetesVersion()) networking: podSubnet: \(podSubnet) diff --git a/Tests/IntegrationTests/K8s/TestK8sRunSerial.swift b/Tests/IntegrationTests/K8s/TestK8sRunSerial.swift index 04f17e02d..7ec8ba152 100644 --- a/Tests/IntegrationTests/K8s/TestK8sRunSerial.swift +++ b/Tests/IntegrationTests/K8s/TestK8sRunSerial.swift @@ -115,4 +115,53 @@ struct TestK8sRunSerial { #expect(server1 != server2) } } + + @Test func testRestartAfterAddressRotation() async throws { + try await ContainerFixture.with { f in + let name = "k8s-\(f.testID)" + let bumper = "\(name)-bumper" + f.addCleanup { try f.doRemoveIfExists(bumper, force: true, ignoreFailure: true) } + f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) } + + try f.restoreWarmupImage(.kindestNodeV1_35_5) + try f.run(["k8s", "create", "--name", name]).check() + + let originalAddress = try f.run(["exec", name, "cat", "/kind/old-ipv4"]) + try originalAddress.check() + #expect(try f.run(["exec", name, "test", "-f", "/kind/kubeadm.conf"]).status == 0) + try f.run([ + "exec", name, "grep", "-F", "controlPlaneEndpoint: 127.0.0.1:6443", "/kind/kubeadm.conf", + ]).check() + + try f.run(["stop", name]).check() + // Advance the rotating allocator while the node is stopped so its + // restart cannot accidentally reuse the same address. + try f.restoreWarmupImage(.alpine320) + try f.run([ + "run", "--name", bumper, "-d", WarmupImage.alpine320.rawValue, + "sleep", "infinity", + ]).check() + + let restart = try f.run(["k8s", "start", "--name", name]) + if restart.status != 0 { + print("[k8s-run] restart stderr: \(restart.error)") + f.dumpNodeDiagnostics(node: name) + } + try restart.check() + + let rotatedAddress = try f.run(["exec", name, "cat", "/kind/old-ipv4"]) + try rotatedAddress.check() + let originalIP = originalAddress.output.trimmingCharacters(in: .whitespacesAndNewlines) + let rotatedIP = rotatedAddress.output.trimmingCharacters(in: .whitespacesAndNewlines) + print("[k8s-run] restart rotated address \(originalIP) -> \(rotatedIP)") + #expect(originalIP != rotatedIP) + #expect(try f.getContainerStatus(name) == "running") + try f.run([ + "exec", name, "grep", "-F", "advertiseAddress: \(rotatedIP)", "/kind/kubeadm.conf", + ]).check() + try f.run([ + "exec", name, "kubectl", "wait", "--for=condition=Ready", "node", "--all", "--timeout=30s", + ]).check() + } + } } diff --git a/Tests/K8sPluginTests/K8sBootstrapTests.swift b/Tests/K8sPluginTests/K8sBootstrapTests.swift new file mode 100644 index 000000000..7eb743e49 --- /dev/null +++ b/Tests/K8sPluginTests/K8sBootstrapTests.swift @@ -0,0 +1,41 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Testing +import Yams + +@testable import ContainerK8s + +@Suite("K8s bootstrap configuration") +struct K8sBootstrapTests { + @Test func nodeLocalEndpointUsesClusterPort() { + #expect(K8sHelper.nodeLocalControlPlaneEndpoint == "127.0.0.1:6443") + } + + @Test func nodeLocalControlPlaneEndpointIsRendered() { + let yaml = K8sHelper.initConfigYAML( + advertiseAddress: "192.168.64.2", + certSANs: ["127.0.0.1"], + controlPlaneEndpoint: K8sHelper.nodeLocalControlPlaneEndpoint) + + #expect(yaml.contains("controlPlaneEndpoint: 127.0.0.1:6443")) + #expect(yaml.contains("advertiseAddress: 192.168.64.2")) + #expect(yaml.contains(" - 127.0.0.1")) + for document in yaml.components(separatedBy: "\n---\n") { + #expect((try? Yams.load(yaml: document)) != nil) + } + } +}