diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 87448884..801c1e82 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -264,6 +264,16 @@ jobs: run: | export HYPEMAN_TEST_PREWARM_DIR="$HOME/.cache/hypeman-ci/darwin-arm64" make test + - name: Run VZ builder integration test + env: + HYPEMAN_RUN_BUILDER_INTEGRATION_TEST: "1" + run: | + # Self-hosted runners retain Docker layers across jobs. Reclaim them so + # the builder image and two VZ builds have predictable disk headroom. + docker system prune --all --force --volumes + PATH="/opt/homebrew/opt/e2fsprogs/sbin:$PATH" \ + go test -count=1 -tags containers_image_openpgp \ + -run='^TestBuilderPersistentCacheReuse$' -timeout=20m -v ./integration - name: Cleanup if: always() run: | @@ -284,7 +294,25 @@ jobs: go-version: '1.25.4' cache: false - name: Install dependencies - run: brew list caddy &>/dev/null || brew install caddy + run: | + brew list caddy &>/dev/null || brew install caddy + if ! docker info >/dev/null 2>&1; then + colima start || { + colima stop --force || true + colima start + } + fi + for attempt in {1..30}; do + if docker info >/dev/null 2>&1; then + break + fi + echo "waiting for Docker daemon (${attempt}/30)" + sleep 2 + done + docker info >/dev/null + # Self-hosted runners retain Docker layers across jobs. Start the + # install E2E with deterministic headroom for its builder image. + docker system prune --all --force --volumes - name: Run E2E install test run: bash scripts/e2e-install-test.sh - name: Run E2E CLI-only install test diff --git a/Makefile b/Makefile index 188d3340..1f243136 100644 --- a/Makefile +++ b/Makefile @@ -315,8 +315,8 @@ test-linux: ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries # macOS tests (no sudo needed, adds e2fsprogs to PATH) # Uses 'go list' to discover compilable packages, then filters out packages -# whose test files reference Linux-only symbols (network, devices, system/init). -DARWIN_EXCLUDE_PKGS := /lib/network|/lib/devices|/lib/system/init|/cmd/vz-shim +# whose test files reference Linux-only symbols (devices, system/init). +DARWIN_EXCLUDE_PKGS := /lib/devices|/lib/system/init|/cmd/vz-shim test-darwin: build-embedded sign-vz-shim @VERBOSE_FLAG=""; \ if [ -n "$(VERBOSE)" ]; then VERBOSE_FLAG="-v"; fi; \ diff --git a/cmd/api/main.go b/cmd/api/main.go index e86f6377..98d7be72 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -47,6 +47,19 @@ import ( "golang.org/x/sync/errgroup" ) +func timeoutNonStreamingRequests(timeout time.Duration) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + timeoutHandler := middleware.Timeout(timeout)(next) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/logs") || strings.HasSuffix(r.URL.Path, "/events") { + next.ServeHTTP(w, r) + return + } + timeoutHandler.ServeHTTP(w, r) + }) + } +} + func main() { if err := run(); err != nil { slog.Error("application terminated", "error", err) @@ -481,7 +494,11 @@ func run() error { }) } - r.Use(middleware.Timeout(60 * time.Second)) + // Streaming endpoints can remain active for longer than the request timeout. + // In particular, cold builds routinely exceed 60 seconds while continuing + // to emit events; cancelling the SSE request makes the CLI report failure + // even though the build is still running. + r.Use(timeoutNonStreamingRequests(60 * time.Second)) // OpenAPI request validation with authentication validatorOptions := &nethttpmiddleware.Options{ diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 09758521..34dbba42 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -21,6 +21,33 @@ import ( const testJWTSecret = "test-secret-key" +func TestRequestTimeoutSkipsStreamingEndpoints(t *testing.T) { + for _, path := range []string{"/instances/test/logs", "/builds/test/events"} { + t.Run(path, func(t *testing.T) { + handler := timeoutNonStreamingRequests(10 * time.Millisecond)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-r.Context().Done(): + return + case <-time.After(30 * time.Millisecond): + w.WriteHeader(http.StatusNoContent) + } + })) + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, path, nil)) + assert.Equal(t, http.StatusNoContent, recorder.Code) + }) + } +} + +func TestRequestTimeoutStillAppliesToRegularEndpoints(t *testing.T) { + handler := timeoutNonStreamingRequests(10 * time.Millisecond)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-r.Context().Done() + })) + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/health", nil)) + assert.Equal(t, http.StatusGatewayTimeout, recorder.Code) +} + func generateValidJWT(userID string) (string, error) { token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ "sub": userID, diff --git a/cmd/api/wire_gen.go b/cmd/api/wire_gen.go index e0117e39..3fa5198e 100644 --- a/cmd/api/wire_gen.go +++ b/cmd/api/wire_gen.go @@ -63,7 +63,7 @@ func initializeApp() (*application, func(), error) { if err != nil { return nil, nil, err } - buildsManager, err := providers.ProvideBuildManager(paths, config, instancesManager, volumesManager, buildersManager, manager, logger) + buildsManager, err := providers.ProvideBuildManager(paths, config, instancesManager, volumesManager, buildersManager, manager, networkManager, logger) if err != nil { return nil, nil, err } diff --git a/config.example.darwin.yaml b/config.example.darwin.yaml index b888cc6c..54787c17 100644 --- a/config.example.darwin.yaml +++ b/config.example.darwin.yaml @@ -12,7 +12,7 @@ # Key differences from Linux (see config.example.yaml): # - hypervisor.default: Use "vz" (Virtualization.framework) instead of cloud-hypervisor/qemu # - data_dir: Uses macOS conventions (~/Library/Application Support) -# - Network settings: network.bridge_name, subnet_cidr, etc. are IGNORED (vz uses NAT) +# - Network settings: network.bridge_name, subnet_cidr, etc. are IGNORED (Hypeman targets VZ's default 192.168.64.0/24 shared NAT) # - Rate limiting: Not supported on macOS (no tc/HTB equivalent) # - GPU passthrough: Not supported on macOS # ============================================================================= @@ -53,10 +53,14 @@ hypervisor: # ============================================================================= # Network Configuration (DIFFERENT ON MACOS) # ============================================================================= -# On macOS with vz, network is handled automatically via NAT: -# - VMs get IP addresses from 192.168.64.0/24 via DHCP -# - No TAP devices, bridges, or iptables needed -# - The following settings are IGNORED on macOS: +# On macOS with VZ, Hypeman targets Virtualization.framework's default shared NAT: +# - Hypeman statically allocates VM addresses from 192.168.64.0/24 +# - The expected host gateway is 192.168.64.1 +# - A host-level vmnet Shared_Net_Address override is not currently supported +# - Other vmnet clients can receive DHCP leases in the same subnet; avoid running +# overlapping VM workloads when static-address collisions are possible +# - No TAP devices, user-created bridges, or iptables rules are needed +# - The following settings are IGNORED on macOS because VZ owns the network: # network.bridge_name, subnet_cidr, subnet_gateway, uplink_interface network: dns_server: 8.8.8.8 @@ -98,8 +102,9 @@ caddy: # ============================================================================= # Build System Configuration # ============================================================================= -# For builds on macOS with vz, the registry URL needs to be accessible from -# NAT VMs. Since vz uses 192.168.64.0/24 for NAT, the host is at 192.168.64.1. +# For builds on macOS with VZ, the registry URL needs to be accessible from +# NAT VMs. On the default shared NAT targeted by Hypeman, the host is at +# 192.168.64.1. # # IMPORTANT: "host.docker.internal" does NOT work in vz VMs - that's a Docker # Desktop-specific hostname. Use the NAT gateway IP instead. diff --git a/integration/builder_cache_darwin_test.go b/integration/builder_cache_darwin_test.go new file mode 100644 index 00000000..fb16936c --- /dev/null +++ b/integration/builder_cache_darwin_test.go @@ -0,0 +1,93 @@ +//go:build darwin + +package integration + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/hypervisor" +) + +func requireBuilderIntegrationHost(t *testing.T) { + t.Helper() + if os.Getenv("HYPEMAN_RUN_BUILDER_INTEGRATION_TEST") != "1" { + t.Skip("set HYPEMAN_RUN_BUILDER_INTEGRATION_TEST=1 to run the VZ builder integration test") + } + if runtime.GOARCH != "arm64" { + t.Skip("VZ builder integration test requires Apple Silicon") + } + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("builder integration test requires Docker") + } + if err := exec.Command("docker", "info").Run(); err != nil { + t.Skip("builder integration test requires a running Docker daemon") + } +} + +func builderIntegrationDataDir(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("/tmp", "hb-") + if err != nil { + t.Fatalf("create short builder integration data directory: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + return dir +} + +func builderIntegrationPlatformConfig(t *testing.T) (config.NetworkConfig, hypervisor.Type) { + t.Helper() + // Deliberately supply the Linux-shaped defaults from issue #358. VZ must + // still use its platform-effective shared NAT for the builder VM and registry. + return config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + SubnetGateway: "10.100.0.1", + DNSServer: "8.8.8.8", + }, hypervisor.TypeVZ +} + +func builderIntegrationDockerSocket(t *testing.T) string { + t.Helper() + if socket := unixDockerSocket(os.Getenv("DOCKER_HOST")); socket != "" { + return socket + } + if output, err := exec.Command("docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}").Output(); err == nil { + if socket := unixDockerSocket(strings.TrimSpace(string(output))); socket != "" { + return socket + } + } + home, _ := os.UserHomeDir() + for _, candidate := range []string{ + "/var/run/docker.sock", + filepath.Join(home, ".colima", "default", "docker.sock"), + filepath.Join(home, ".docker", "run", "docker.sock"), + } { + if _, err := os.Stat(candidate); err == nil { + return candidate + } + } + t.Fatal("builder integration test requires a local Docker Unix socket") + return "" +} + +func unixDockerSocket(host string) string { + if strings.HasPrefix(host, "unix://") { + return strings.TrimPrefix(host, "unix://") + } + if strings.HasPrefix(host, "/") { + return host + } + return "" +} + +func prepareBuilderIntegrationRegistryAccess(t *testing.T, bridge string) { + t.Helper() + // VZ's shared NAT can reach host listeners through its gateway without a + // host firewall rule managed by Hypeman. +} diff --git a/integration/builder_cache_linux_test.go b/integration/builder_cache_linux_test.go index 690bc9b3..af475a5c 100644 --- a/integration/builder_cache_linux_test.go +++ b/integration/builder_cache_linux_test.go @@ -1,189 +1,44 @@ +//go:build linux + package integration import ( - "archive/tar" - "bytes" - "compress/gzip" - "context" - "crypto/rand" - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "math/big" - "net" - "net/http" "os" "os/exec" - "strconv" "strings" "testing" - "time" "github.com/kernel/hypeman/cmd/api/config" - "github.com/kernel/hypeman/lib/builders" - "github.com/kernel/hypeman/lib/builds" - "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/images" - "github.com/kernel/hypeman/lib/instances" - "github.com/kernel/hypeman/lib/network" - "github.com/kernel/hypeman/lib/paths" - "github.com/kernel/hypeman/lib/registry" - "github.com/kernel/hypeman/lib/system" - "github.com/kernel/hypeman/lib/volumes" - "github.com/stretchr/testify/assert" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/stretchr/testify/require" ) -func TestBuilderPersistentCacheReuse(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test in short mode") - } +func requireBuilderIntegrationHost(t *testing.T) { + t.Helper() if os.Geteuid() != 0 { - t.Skip("builder integration test requires root") + t.Skip("builder integration test requires root on Linux") } if _, err := os.Stat("/dev/kvm"); err != nil { t.Skip("builder integration test requires /dev/kvm") } +} - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) - defer cancel() - t.Chdir("..") - - p := paths.New(t.TempDir()) - cfg := &config.Config{ - DataDir: p.DataDir(), - Network: newParallelTestNetworkConfig(t), - } - gateway, err := network.DeriveGateway(cfg.Network.SubnetCIDR) - require.NoError(t, err) - cfg.Network.SubnetGateway = gateway - - imageManager, err := images.NewManager(p, 1, nil) - require.NoError(t, err) - volumeManager := volumes.NewManager(p, 0, nil) - networkManager := network.NewManager(p, cfg, nil) - require.NoError(t, networkManager.Initialize(ctx, nil)) - allowHostRegistryTraffic(t, cfg.Network.BridgeName) - systemManager := system.NewManager(p) - require.NoError(t, systemManager.EnsureSystemFiles(ctx)) - instanceManager := instances.NewManager( - p, - imageManager, - systemManager, - networkManager, - devices.NewManager(p), - volumeManager, - instances.ResourceLimits{MaxOverlaySize: 100 << 30}, - "", - instances.SnapshotPolicy{}, - nil, - nil, - ) - t.Cleanup(func() { - all, listErr := instanceManager.ListInstances(context.Background(), nil) - if listErr != nil { - t.Logf("list instances during cleanup: %v", listErr) - return - } - for _, instance := range all { - if deleteErr := instanceManager.DeleteInstance(context.Background(), instance.Id); deleteErr != nil { - t.Logf("delete instance %s during cleanup: %v", instance.Id, deleteErr) - } - } - }) - - registryURL, registryCA := startBuildRegistry(t, gateway, p, imageManager) - - builderManager, err := builders.NewManager( - p, - builders.Config{DefaultDiskSizeGb: 4}, - volumeManager, - instanceManager, - nil, - nil, - ) - require.NoError(t, err) - require.NoError(t, builderManager.Start(ctx)) - - buildManager, err := builds.NewManager( - p, - builds.Config{ - MaxConcurrentBuilds: 1, - RegistryURL: registryURL, - RegistryCACert: registryCA, - RegistrySecret: "builder-cache-integration-test", - DefaultTimeout: 600, - }, - instanceManager, - volumeManager, - builderManager, - imageManager, - nil, - nil, - nil, - ) - require.NoError(t, err) - builderManager.SetBuildActivityChecker(buildManager.BuilderHasBuilds) - require.NoError(t, buildManager.Start(ctx)) - require.EventuallyWithT(t, func(collect *assert.CollectT) { - all, listErr := imageManager.ListImages(ctx) - require.NoError(collect, listErr) - ready := false - for _, image := range all { - if strings.Contains(image.Name, "/internal/builder") { - ready = image.Status == images.StatusReady - } - } - require.True(collect, ready) - }, 5*time.Minute, time.Second) - require.Eventually(t, buildManager.ReadyForBuilds, time.Second, 10*time.Millisecond) - - builder, err := builderManager.CreateBuilder(ctx, builders.CreateBuilderRequest{DiskSizeGb: 4}) - require.NoError(t, err) - - dockerfile := `FROM alpine:3.18 -ARG CACHE_BUSTER -RUN --mount=type=cache,target=/cache sh -c 'if [ -f /cache/sentinel ]; then echo BUILDER_CACHE_HIT; else echo BUILDER_CACHE_MISS; touch /cache/sentinel; fi; echo "$CACHE_BUSTER" > /cache-buster' -` - source := sourceArchive(t, dockerfile) - first := runBuilderBuild(t, ctx, buildManager, builder.ID, dockerfile, source, "first") - firstLogs, err := buildManager.GetBuildLogs(ctx, first.ID) - require.NoError(t, err) - require.Contains(t, string(firstLogs), "BUILDER_CACHE_MISS") +func builderIntegrationDataDir(t *testing.T) string { + t.Helper() + return t.TempDir() +} - second := runBuilderBuild(t, ctx, buildManager, builder.ID, dockerfile, source, "second") - secondLogs, err := buildManager.GetBuildLogs(ctx, second.ID) - require.NoError(t, err) - require.Contains(t, string(secondLogs), "BUILDER_CACHE_HIT") - require.NotNil(t, first.BuilderInstanceID) - require.NotNil(t, second.BuilderInstanceID) - require.NotEqual(t, *first.BuilderInstanceID, *second.BuilderInstanceID) +func builderIntegrationPlatformConfig(t *testing.T) (config.NetworkConfig, hypervisor.Type) { + t.Helper() + return newParallelTestNetworkConfig(t), hypervisor.TypeCloudHypervisor } -func startBuildRegistry(t *testing.T, gateway string, p *paths.Paths, imageManager images.Manager) (string, string) { +func builderIntegrationDockerSocket(t *testing.T) string { t.Helper() - reg, err := registry.New(p, imageManager) - require.NoError(t, err) - certPEM, keyPEM := registryCertificate(t, net.ParseIP(gateway)) - certificate, err := tls.X509KeyPair(certPEM, keyPEM) - require.NoError(t, err) - listener, err := net.Listen("tcp", "0.0.0.0:0") - require.NoError(t, err) - tlsListener := tls.NewListener(listener, &tls.Config{Certificates: []tls.Certificate{certificate}, MinVersion: tls.VersionTLS12}) - server := &http.Server{Handler: reg.Handler()} - go func() { - if serveErr := server.Serve(tlsListener); serveErr != nil && serveErr != http.ErrServerClosed { - t.Logf("registry server: %v", serveErr) - } - }() - t.Cleanup(func() { _ = server.Close() }) - port := strconv.Itoa(listener.Addr().(*net.TCPAddr).Port) - return net.JoinHostPort(gateway, port), string(certPEM) + return "/var/run/docker.sock" } -func allowHostRegistryTraffic(t *testing.T, bridge string) { +func prepareBuilderIntegrationRegistryAccess(t *testing.T, bridge string) { t.Helper() if exec.Command("nft", "list", "table", "inet", "kernel_firewall").Run() != nil { return @@ -206,66 +61,3 @@ func allowHostRegistryTraffic(t *testing.T, bridge string) { } }) } - -func registryCertificate(t *testing.T, ip net.IP) ([]byte, []byte) { - t.Helper() - key, err := rsa.GenerateKey(rand.Reader, 2048) - require.NoError(t, err) - now := time.Now() - template := &x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{CommonName: ip.String()}, - NotBefore: now.Add(-time.Minute), - NotAfter: now.Add(time.Hour), - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - IPAddresses: []net.IP{ip}, - } - der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) - require.NoError(t, err) - certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) - keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) - return certPEM, keyPEM -} - -func runBuilderBuild(t *testing.T, ctx context.Context, manager builds.Manager, builderID, dockerfile string, source []byte, cacheBuster string) *builds.Build { - t.Helper() - build, err := manager.CreateBuild(ctx, builds.CreateBuildRequest{ - Dockerfile: dockerfile, - BuilderID: builderID, - BuildArgs: map[string]string{"CACHE_BUSTER": cacheBuster}, - }, source) - require.NoError(t, err) - - require.EventuallyWithT(t, func(collect *assert.CollectT) { - current, getErr := manager.GetBuild(ctx, build.ID) - require.NoError(collect, getErr) - require.Contains(collect, []string{builds.StatusReady, builds.StatusFailed}, current.Status) - }, 10*time.Minute, time.Second) - - result, err := manager.GetBuild(ctx, build.ID) - require.NoError(t, err) - if result.Status != builds.StatusReady { - logs, _ := manager.GetBuildLogs(ctx, build.ID) - t.Fatalf("build %s failed: %v\n%s", build.ID, result.Error, logs) - } - return result -} - -func sourceArchive(t *testing.T, dockerfile string) []byte { - t.Helper() - var out bytes.Buffer - gz := gzip.NewWriter(&out) - tw := tar.NewWriter(gz) - contents := []byte(dockerfile) - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: "Dockerfile", - Mode: 0644, - Size: int64(len(contents)), - })) - _, err := tw.Write(contents) - require.NoError(t, err) - require.NoError(t, tw.Close()) - require.NoError(t, gz.Close()) - return out.Bytes() -} diff --git a/integration/builder_cache_test.go b/integration/builder_cache_test.go new file mode 100644 index 00000000..9dd798c3 --- /dev/null +++ b/integration/builder_cache_test.go @@ -0,0 +1,282 @@ +//go:build linux || darwin + +package integration + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/builders" + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/registry" + "github.com/kernel/hypeman/lib/system" + "github.com/kernel/hypeman/lib/volumes" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuilderPersistentCacheReuse(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + requireBuilderIntegrationHost(t) + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + defer cancel() + repoRoot, err := filepath.Abs("..") + require.NoError(t, err) + t.Chdir(builderIntegrationDockerContext(t, repoRoot)) + + p := paths.New(builderIntegrationDataDir(t)) + networkConfig, defaultHypervisor := builderIntegrationPlatformConfig(t) + cfg := &config.Config{ + DataDir: p.DataDir(), + Network: networkConfig, + } + + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + volumeManager := volumes.NewManager(p, 0, nil) + networkManager := network.NewManager(p, cfg, nil) + require.NoError(t, networkManager.Initialize(ctx, nil)) + effectiveNetwork, err := networkManager.EffectiveDefaultNetwork() + require.NoError(t, err) + prepareBuilderIntegrationRegistryAccess(t, effectiveNetwork.Bridge) + systemManager := system.NewManager(p) + require.NoError(t, systemManager.EnsureSystemFiles(ctx)) + instanceManager := instances.NewManager( + p, + imageManager, + systemManager, + networkManager, + devices.NewManager(p), + volumeManager, + instances.ResourceLimits{MaxOverlaySize: 100 << 30}, + defaultHypervisor, + instances.SnapshotPolicy{}, + nil, + nil, + ) + t.Cleanup(func() { + all, listErr := instanceManager.ListInstances(context.Background(), nil) + if listErr != nil { + t.Logf("list instances during cleanup: %v", listErr) + return + } + for _, instance := range all { + if deleteErr := instanceManager.DeleteInstance(context.Background(), instance.Id); deleteErr != nil { + t.Logf("delete instance %s during cleanup: %v", instance.Id, deleteErr) + } + } + }) + + registryURL, registryCA := startBuildRegistry(t, effectiveNetwork.Gateway, p, imageManager) + + builderManager, err := builders.NewManager( + p, + builders.Config{DefaultDiskSizeGb: 4}, + volumeManager, + instanceManager, + nil, + nil, + ) + require.NoError(t, err) + require.NoError(t, builderManager.Start(ctx)) + + buildManager, err := builds.NewManager( + p, + builds.Config{ + MaxConcurrentBuilds: 1, + DockerSocket: builderIntegrationDockerSocket(t), + RegistryURL: registryURL, + RegistryCACert: registryCA, + RegistrySecret: "builder-cache-integration-test", + DefaultTimeout: 600, + }, + instanceManager, + volumeManager, + builderManager, + imageManager, + nil, + nil, + nil, + ) + require.NoError(t, err) + builderManager.SetBuildActivityChecker(buildManager.BuilderHasBuilds) + require.NoError(t, buildManager.Start(ctx)) + require.EventuallyWithT(t, func(collect *assert.CollectT) { + all, listErr := imageManager.ListImages(ctx) + require.NoError(collect, listErr) + ready := false + for _, image := range all { + if strings.Contains(image.Name, "/internal/builder") { + ready = image.Status == images.StatusReady + } + } + require.True(collect, ready) + }, 5*time.Minute, time.Second) + require.Eventually(t, buildManager.ReadyForBuilds, 30*time.Second, 100*time.Millisecond) + + builder, err := builderManager.CreateBuilder(ctx, builders.CreateBuilderRequest{DiskSizeGb: 4}) + require.NoError(t, err) + + dockerfile := `FROM alpine:3.18 +ARG CACHE_BUSTER +RUN --mount=type=cache,target=/cache sh -c 'if [ -f /cache/sentinel ]; then echo BUILDER_CACHE_HIT; else echo BUILDER_CACHE_MISS; touch /cache/sentinel; fi; echo "$CACHE_BUSTER" > /cache-buster' +` + source := sourceArchive(t, dockerfile) + first := runBuilderBuild(t, ctx, buildManager, builder.ID, dockerfile, source, "first") + firstLogs, err := buildManager.GetBuildLogs(ctx, first.ID) + require.NoError(t, err) + require.Contains(t, string(firstLogs), "BUILDER_CACHE_MISS") + + second := runBuilderBuild(t, ctx, buildManager, builder.ID, dockerfile, source, "second") + secondLogs, err := buildManager.GetBuildLogs(ctx, second.ID) + require.NoError(t, err) + require.Contains(t, string(secondLogs), "BUILDER_CACHE_HIT") + require.NotNil(t, first.BuilderInstanceID) + require.NotNil(t, second.BuilderInstanceID) + require.NotEqual(t, *first.BuilderInstanceID, *second.BuilderInstanceID) +} + +func builderIntegrationDockerContext(t *testing.T, repoRoot string) string { + t.Helper() + contextDir := t.TempDir() + for _, path := range []string{"go.mod", "go.sum", "lib/guest/guest.pb.go", "lib/guest/guest_grpc.pb.go"} { + copyBuilderIntegrationFile(t, repoRoot, contextDir, path) + } + for _, dir := range []string{"lib/builds/builder_agent", "lib/system/guest_agent"} { + err := filepath.WalkDir(filepath.Join(repoRoot, dir), func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || filepath.Ext(path) != ".go" || strings.HasSuffix(path, "_test.go") { + return nil + } + rel, err := filepath.Rel(repoRoot, path) + if err != nil { + return err + } + copyBuilderIntegrationFile(t, repoRoot, contextDir, rel) + return nil + }) + require.NoError(t, err) + } + return contextDir +} + +func copyBuilderIntegrationFile(t *testing.T, sourceRoot, destinationRoot, path string) { + t.Helper() + contents, err := os.ReadFile(filepath.Join(sourceRoot, path)) + require.NoError(t, err) + destination := filepath.Join(destinationRoot, path) + require.NoError(t, os.MkdirAll(filepath.Dir(destination), 0o755)) + require.NoError(t, os.WriteFile(destination, contents, 0o644)) +} + +func startBuildRegistry(t *testing.T, gateway string, p *paths.Paths, imageManager images.Manager) (string, string) { + t.Helper() + reg, err := registry.New(p, imageManager) + require.NoError(t, err) + certPEM, keyPEM := registryCertificate(t, net.ParseIP(gateway)) + certificate, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + listener, err := net.Listen("tcp", "0.0.0.0:0") + require.NoError(t, err) + tlsListener := tls.NewListener(listener, &tls.Config{Certificates: []tls.Certificate{certificate}, MinVersion: tls.VersionTLS12}) + server := &http.Server{Handler: reg.Handler()} + go func() { + if serveErr := server.Serve(tlsListener); serveErr != nil && serveErr != http.ErrServerClosed { + t.Logf("registry server: %v", serveErr) + } + }() + t.Cleanup(func() { _ = server.Close() }) + port := strconv.Itoa(listener.Addr().(*net.TCPAddr).Port) + return net.JoinHostPort(gateway, port), string(certPEM) +} + +func registryCertificate(t *testing.T, ip net.IP) ([]byte, []byte) { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + now := time.Now() + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: ip.String()}, + NotBefore: now.Add(-time.Minute), + NotAfter: now.Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + IPAddresses: []net.IP{ip}, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + require.NoError(t, err) + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + return certPEM, keyPEM +} + +func runBuilderBuild(t *testing.T, ctx context.Context, manager builds.Manager, builderID, dockerfile string, source []byte, cacheBuster string) *builds.Build { + t.Helper() + build, err := manager.CreateBuild(ctx, builds.CreateBuildRequest{ + Dockerfile: dockerfile, + BuilderID: builderID, + BuildArgs: map[string]string{"CACHE_BUSTER": cacheBuster}, + }, source) + require.NoError(t, err) + + require.EventuallyWithT(t, func(collect *assert.CollectT) { + current, getErr := manager.GetBuild(ctx, build.ID) + require.NoError(collect, getErr) + require.Contains(collect, []string{builds.StatusReady, builds.StatusFailed}, current.Status) + }, 10*time.Minute, time.Second) + + result, err := manager.GetBuild(ctx, build.ID) + require.NoError(t, err) + if result.Status != builds.StatusReady { + logs, _ := manager.GetBuildLogs(ctx, build.ID) + t.Fatalf("build %s failed: %v\n%s", build.ID, result.Error, logs) + } + return result +} + +func sourceArchive(t *testing.T, dockerfile string) []byte { + t.Helper() + var out bytes.Buffer + gz := gzip.NewWriter(&out) + tw := tar.NewWriter(gz) + contents := []byte(dockerfile) + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "Dockerfile", + Mode: 0644, + Size: int64(len(contents)), + })) + _, err := tw.Write(contents) + require.NoError(t, err) + require.NoError(t, tw.Close()) + require.NoError(t, gz.Close()) + return out.Bytes() +} diff --git a/lib/builds/manager.go b/lib/builds/manager.go index fa2153de..b9157d93 100644 --- a/lib/builds/manager.go +++ b/lib/builds/manager.go @@ -32,8 +32,9 @@ import ( const ( // releaseBuildMaxAttempts bounds the builder-release retry at the end // of a build; releaseBuildRetryDelay spaces the attempts. - releaseBuildMaxAttempts = 5 - releaseBuildRetryDelay = time.Second + releaseBuildMaxAttempts = 5 + releaseBuildRetryDelay = time.Second + builderImageRetryInterval = 5 * time.Second ) //go:embed images/generic/Dockerfile @@ -205,10 +206,21 @@ func NewManager( // Start starts the build manager's background services func (m *manager) Start(ctx context.Context) error { go func() { - m.ensureBuilderImage(ctx) - // Recover pending builds only after the builder image is ready, - // otherwise recovered builds fail with "builder image is being prepared". - m.RecoverPendingBuilds() + for { + m.ensureBuilderImage(ctx) + // Recover pending builds only after the builder image is ready, + // otherwise recovered builds fail with "builder image is being prepared". + if m.ReadyForBuilds() { + m.RecoverPendingBuilds() + return + } + m.logger.Warn("builder image preparation failed; retrying", "retry_in", builderImageRetryInterval) + select { + case <-ctx.Done(): + return + case <-time.After(builderImageRetryInterval): + } + } }() m.logger.Info("build manager started") return nil @@ -229,16 +241,30 @@ func (m *manager) ReadyForBuilds() bool { // // This runs in a background goroutine during startup. func (m *manager) ensureBuilderImage(ctx context.Context) { - defer m.builderReady.Store(true) - if m.config.BuilderImage != "" { - // Explicit builder image configured - check if already available - if _, err := m.imageManager.GetImage(ctx, m.config.BuilderImage); err == nil { - m.logger.Info("builder image already available", "image", m.config.BuilderImage) - return + // Explicit builder image configured - check if already available. + if image, err := m.imageManager.GetImage(ctx, m.config.BuilderImage); err == nil { + switch image.Status { + case images.StatusReady: + m.logger.Info("builder image already available", "image", m.config.BuilderImage) + m.builderReady.Store(true) + return + case images.StatusFailed: + // CreateImage removes failed image state and queues a fresh pull. + m.logger.Info("retrying failed builder image", "image", m.config.BuilderImage) + default: + m.logger.Info("waiting for existing builder image", "image", m.config.BuilderImage, "status", image.Status) + if err := m.waitForBuilderImageReady(ctx, m.config.BuilderImage); err != nil { + m.logger.Warn("builder image failed to become ready", "image", m.config.BuilderImage, "error", err) + return + } + m.builderReady.Store(true) + return + } } - // Not in store - try to pull it from remote registry + // Missing or failed: pull it from the remote registry. CreateImage cleans + // failed metadata before re-queueing the conversion. m.logger.Info("pulling builder image", "image", m.config.BuilderImage) if _, err := m.imageManager.CreateImage(ctx, images.CreateImageRequest{ Name: m.config.BuilderImage, @@ -248,7 +274,9 @@ func (m *manager) ensureBuilderImage(ctx context.Context) { } if err := m.waitForBuilderImageReady(ctx, m.config.BuilderImage); err != nil { m.logger.Warn("builder image failed to become ready", "image", m.config.BuilderImage, "error", err) + return } + m.builderReady.Store(true) return } @@ -260,6 +288,7 @@ func (m *manager) ensureBuilderImage(ctx context.Context) { return } m.config.BuilderImage = imageRef + m.builderReady.Store(true) m.logger.Info("builder image ready", "image", imageRef) } @@ -268,7 +297,8 @@ func (m *manager) ensureBuilderImage(ctx context.Context) { // // The flow is: // 1. Write embedded Dockerfile to a temp directory -// 2. Build with Docker (uses cwd as context for COPY directives) +// 2. Build with Docker when running from a source checkout, or reuse the +// installer-built local image when no source context is available // 3. Export with docker save to a tarball // 4. Load tarball with go-containerregistry and write to the shared OCI layout cache // 5. Call ImportLocalImage to trigger ext4 conversion @@ -298,20 +328,32 @@ func (m *manager) buildBuilderFromDockerfile(ctx context.Context) (string, error return "", fmt.Errorf("write Dockerfile: %w", err) } - // Build with Docker (context is cwd = repo root in development) - localTag := fmt.Sprintf("hypeman-builder-tmp:%d", time.Now().Unix()) - m.logger.Info("building builder image with Docker", "tag", localTag) + // Development runs have the source checkout needed by the Dockerfile's COPY + // directives. Installed launchd services start outside that checkout, so the + // installer builds this image before loading the service. + localTag := "hypeman/builder:latest" + if _, err := os.Stat("go.mod"); err == nil { + localTag = fmt.Sprintf("hypeman-builder-tmp:%d", time.Now().Unix()) + m.logger.Info("building builder image with Docker", "tag", localTag) - buildCmd := exec.CommandContext(ctx, "docker", "build", "-t", localTag, "-f", dockerfilePath, ".") - buildCmd.Env = dockerEnv - if output, err := buildCmd.CombinedOutput(); err != nil { - return "", fmt.Errorf("docker build: %s: %w", string(output), err) + buildCmd := exec.CommandContext(ctx, "docker", "build", "-t", localTag, "-f", dockerfilePath, ".") + buildCmd.Env = dockerEnv + if output, err := buildCmd.CombinedOutput(); err != nil { + return "", fmt.Errorf("docker build: %s: %w", string(output), err) + } + defer func() { + rmCmd := exec.Command("docker", "rmi", localTag) + rmCmd.Env = dockerEnv + rmCmd.Run() + }() + } else { + inspectCmd := exec.CommandContext(ctx, "docker", "image", "inspect", localTag) + inspectCmd.Env = dockerEnv + if output, inspectErr := inspectCmd.CombinedOutput(); inspectErr != nil { + return "", fmt.Errorf("source checkout unavailable and local builder image %s not found: %s: %w", localTag, string(output), inspectErr) + } + m.logger.Info("using installer-built local builder image", "tag", localTag) } - defer func() { - rmCmd := exec.Command("docker", "rmi", localTag) - rmCmd.Env = dockerEnv - rmCmd.Run() - }() // Export image to tarball (avoids docker push) tarPath := filepath.Join(tmpDir, "builder.tar") diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index 1b69be16..dfdc9fca 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -309,9 +309,10 @@ func (m *mockSecretProvider) GetSecrets(ctx context.Context, secretIDs []string) // mockImageManager implements images.Manager for testing type mockImageManager struct { - mu sync.RWMutex - images map[string]*images.Image - getImageErr error + mu sync.RWMutex + images map[string]*images.Image + createImageFunc func(ctx context.Context, req images.CreateImageRequest) (*images.Image, error) + getImageErr error } func newMockImageManager() *mockImageManager { @@ -329,6 +330,9 @@ func (m *mockImageManager) ListImages(ctx context.Context) ([]images.Image, erro } func (m *mockImageManager) CreateImage(ctx context.Context, req images.CreateImageRequest) (*images.Image, error) { + if m.createImageFunc != nil { + return m.createImageFunc(ctx, req) + } img := &images.Image{ Name: req.Name, Status: images.StatusPending, @@ -481,6 +485,62 @@ func setupTestManagerWithImageMgr(t *testing.T) (*manager, *mockInstanceManager, return mgr, instanceMgr, volumeMgr, imageMgr, tempDir } +func TestEnsureBuilderImageOnlyMarksReadyAfterSuccess(t *testing.T) { + t.Run("existing image", func(t *testing.T) { + mgr, _, _, imageMgr, tempDir := setupTestManagerWithImageMgr(t) + defer os.RemoveAll(tempDir) + mgr.builderReady.Store(false) + imageMgr.images[mgr.config.BuilderImage] = &images.Image{ + Name: mgr.config.BuilderImage, + Status: images.StatusReady, + } + + mgr.ensureBuilderImage(context.Background()) + + assert.True(t, mgr.ReadyForBuilds()) + }) + + t.Run("existing pending image never becomes ready", func(t *testing.T) { + mgr, _, _, imageMgr, tempDir := setupTestManagerWithImageMgr(t) + defer os.RemoveAll(tempDir) + mgr.builderReady.Store(false) + imageMgr.images[mgr.config.BuilderImage] = &images.Image{ + Name: mgr.config.BuilderImage, + Status: images.StatusPending, + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + mgr.ensureBuilderImage(ctx) + + assert.False(t, mgr.ReadyForBuilds()) + }) + + t.Run("existing failed image is requeued", func(t *testing.T) { + mgr, _, _, imageMgr, tempDir := setupTestManagerWithImageMgr(t) + defer os.RemoveAll(tempDir) + mgr.builderReady.Store(false) + imageMgr.images[mgr.config.BuilderImage] = &images.Image{ + Name: mgr.config.BuilderImage, + Status: images.StatusFailed, + } + createCalls := 0 + imageMgr.createImageFunc = func(_ context.Context, req images.CreateImageRequest) (*images.Image, error) { + createCalls++ + imageMgr.mu.Lock() + defer imageMgr.mu.Unlock() + image := &images.Image{Name: req.Name, Status: images.StatusReady} + imageMgr.images[req.Name] = image + return image, nil + } + + mgr.ensureBuilderImage(context.Background()) + + assert.Equal(t, 1, createCalls) + assert.True(t, mgr.ReadyForBuilds()) + }) +} + func TestCreateBuild_Success(t *testing.T) { mgr, _, _, tempDir := setupTestManager(t) defer os.RemoveAll(tempDir) diff --git a/lib/network/README.md b/lib/network/README.md index 2ed3dfe8..098caf84 100644 --- a/lib/network/README.md +++ b/lib/network/README.md @@ -7,9 +7,9 @@ Manages the default virtual network for instances. | Platform | Network Model | Implementation | |----------|---------------|----------------| | Linux | Bridge + TAP | Linux bridge with TAP devices per VM, iptables NAT | -| macOS | NAT | Virtualization.framework built-in NAT (192.168.64.0/24) | +| macOS | NAT | Virtualization.framework shared NAT (default: 192.168.64.0/24) | -On macOS, the network manager skips bridge/TAP creation since vz provides NAT networking automatically. +On macOS, the network manager skips bridge/TAP creation because VZ provides the NAT network. Hypeman currently targets vmnet's default shared subnet, statically allocating guest addresses from `192.168.64.0/24` with gateway `192.168.64.1`; Linux bridge/subnet configuration does not override those values. Host-level `Shared_Net_Address` overrides are not currently supported. Other vmnet clients may receive DHCP leases in the same subnet, so concurrent tools such as Colima, Docker, or UTM can create an address-collision risk that Hypeman's allocator cannot detect. --- @@ -111,10 +111,11 @@ Hypeman provides a single default network that all instances can optionally conn ### Default Network -- Auto-created on first `Initialize()` call -- Configured from environment variables (BRIDGE_NAME, SUBNET_CIDR, SUBNET_GATEWAY) +- Initialized on the first `Initialize()` call +- Linux creates it from bridge/subnet configuration +- macOS targets VZ's default `192.168.64.0/24` shared NAT network regardless of Linux bridge/subnet configuration - Named "default" (only network in the system) -- Always uses bridge_slave isolated mode for VM-to-VM isolation +- Linux uses bridge_slave isolated mode for VM-to-VM isolation ### Name Uniqueness diff --git a/lib/network/bridge_darwin.go b/lib/network/bridge_darwin.go index c6e09cc3..50799f43 100644 --- a/lib/network/bridge_darwin.go +++ b/lib/network/bridge_darwin.go @@ -58,16 +58,10 @@ func (m *manager) tapDeviceExists(tapName string) bool { return true } -// queryNetworkState returns a stub network state for macOS. -// On macOS, we use NAT which doesn't have a physical bridge. +// queryNetworkState returns the default vmnet shared-NAT network targeted by +// Hypeman. Host-level Shared_Net_Address overrides are not currently supported. func (m *manager) queryNetworkState(bridgeName string) (*Network, error) { - // Return a virtual network representing macOS NAT - // The actual IP will be assigned by Virtualization.framework's DHCP - return &Network{ - Bridge: "nat", - Gateway: "192.168.64.1", // Default macOS vz NAT gateway - Subnet: "192.168.64.0/24", - }, nil + return newDefaultNetwork(vzNATBridge, vzNATSubnet, vzNATGateway), nil } // CleanupOrphanedTAPs is a no-op on macOS as we don't create TAP devices. diff --git a/lib/network/bridge_linux.go b/lib/network/bridge_linux.go index 31297826..96b7aef0 100644 --- a/lib/network/bridge_linux.go +++ b/lib/network/bridge_linux.go @@ -128,13 +128,13 @@ func (m *manager) createBridge(ctx context.Context, name, gateway, subnet string } expectedGW := net.ParseIP(gateway) - hasExpectedIP := false - var actualIPs []string + if expectedGW == nil { + return fmt.Errorf("invalid gateway IP: %s", gateway) + } + hasExpectedIP, hasExpectedMask := bridgeAddressMatches(addrs, expectedGW, ipNet.Mask) + actualIPs := make([]string, 0, len(addrs)) for _, addr := range addrs { - actualIPs = append(actualIPs, addr.IPNet.String()) - if addr.IP.Equal(expectedGW) { - hasExpectedIP = true - } + actualIPs = append(actualIPs, addr.String()) } if !hasExpectedIP { @@ -145,6 +145,14 @@ func (m *manager) createBridge(ctx context.Context, name, gateway, subnet string "or (3) delete the bridge with: sudo ip link delete %s", name, actualIPs, gateway, ones, name) } + if !hasExpectedMask { + ones, _ := ipNet.Mask.Size() + return fmt.Errorf("bridge %s exists with gateway %s but its prefix does not match /%d (addresses: %v). "+ + "Options: (1) update SUBNET_CIDR to match the existing bridge, "+ + "(2) use a different BRIDGE_NAME, "+ + "or (3) delete the bridge with: sudo ip link delete %s", + name, gateway, ones, actualIPs, name) + } // Bridge exists with correct IP, verify it's up if err := netlink.LinkSetUp(existing); err != nil { @@ -1064,42 +1072,77 @@ func (m *manager) tapDeviceExists(tapName string) bool { return err == nil } -// queryNetworkState queries kernel for bridge state +// queryNetworkState queries kernel for bridge state. func (m *manager) queryNetworkState(bridgeName string) (*Network, error) { link, err := netlink.LinkByName(bridgeName) if err != nil { - return nil, ErrNotFound + return nil, fmt.Errorf("%w: look up bridge %q: %v", ErrNotFound, bridgeName, err) } - // Verify it's actually a bridge if link.Type() != "bridge" { return nil, fmt.Errorf("link %s is not a bridge", bridgeName) } - // Get IP addresses - addrs, err := netlink.AddrList(link, netlink.FAMILY_V4) + addrs, err := listBridgeAddrsWithRetry(link) if err != nil { return nil, fmt.Errorf("list addresses: %w", err) } - if len(addrs) == 0 { return nil, fmt.Errorf("bridge has no IP addresses") } - // Use first IP as gateway - gateway := addrs[0].IP.String() - subnet := addrs[0].IPNet.String() - - // Bridge exists and has IP - that's sufficient - // OperState can be OperUp, OperUnknown, etc. - all are functional for our purposes + // A bridge may have multiple IPv4 addresses. Prefer the configured gateway + // when createBridge has confirmed that it is present rather than depending on + // netlink address ordering. + gatewayAddr := addrs[0] + requestedNetwork, requestedErr := m.platformDefaultNetwork() + if requestedErr == nil { + gatewayAddr = selectBridgeGatewayAddr(addrs, net.ParseIP(requestedNetwork.Gateway)) + } + // Bridge existence plus an IPv4 address is sufficient. OperState may be + // OperUp or OperUnknown; both are functional for this bridge. return &Network{ Bridge: bridgeName, - Gateway: gateway, - Subnet: subnet, + Gateway: gatewayAddr.IP.String(), + Subnet: canonicalSubnetCIDR(gatewayAddr.IPNet), }, nil } +func bridgeAddressMatches(addrs []netlink.Addr, expectedGateway net.IP, expectedMask net.IPMask) (hasGateway, hasMatchingMask bool) { + expectedOnes, expectedBits := expectedMask.Size() + for _, addr := range addrs { + if !addr.IP.Equal(expectedGateway) { + continue + } + hasGateway = true + if addr.IPNet == nil { + continue + } + ones, bits := addr.IPNet.Mask.Size() + if ones == expectedOnes && bits == expectedBits { + hasMatchingMask = true + } + } + return hasGateway, hasMatchingMask +} + +func canonicalSubnetCIDR(ipNet *net.IPNet) string { + if ipNet == nil { + return "" + } + return (&net.IPNet{IP: ipNet.IP.Mask(ipNet.Mask), Mask: ipNet.Mask}).String() +} + +func selectBridgeGatewayAddr(addrs []netlink.Addr, preferredGateway net.IP) netlink.Addr { + for _, addr := range addrs { + if addr.IP.Equal(preferredGateway) { + return addr + } + } + return addrs[0] +} + // CleanupOrphanedTAPs removes TAP devices that aren't used by any running instance. // preserveInstanceIDs is the authoritative set of instance IDs whose TAPs must be // kept. Pass nil or an empty list to skip cleanup entirely (used when we couldn't diff --git a/lib/network/default_network_darwin.go b/lib/network/default_network_darwin.go new file mode 100644 index 00000000..3b10f950 --- /dev/null +++ b/lib/network/default_network_darwin.go @@ -0,0 +1,16 @@ +//go:build darwin + +package network + +const ( + vzNATBridge = "nat" + vzNATSubnet = "192.168.64.0/24" + vzNATGateway = "192.168.64.1" +) + +// platformDefaultNetwork returns the default vmnet shared-NAT network targeted +// by Hypeman. Linux bridge settings do not change the network attached to VZ +// guests; host-level Shared_Net_Address overrides are not currently supported. +func (m *manager) platformDefaultNetwork() (*Network, error) { + return newDefaultNetwork(vzNATBridge, vzNATSubnet, vzNATGateway), nil +} diff --git a/lib/network/default_network_linux.go b/lib/network/default_network_linux.go new file mode 100644 index 00000000..82514d6b --- /dev/null +++ b/lib/network/default_network_linux.go @@ -0,0 +1,25 @@ +//go:build linux + +package network + +import "fmt" + +// platformDefaultNetwork returns the network requested by the Linux bridge +// configuration. Initialize verifies and creates this network before caching the +// bridge state reported by the kernel. +func (m *manager) platformDefaultNetwork() (*Network, error) { + gateway := m.config.Network.SubnetGateway + if gateway == "" { + var err error + gateway, err = DeriveGateway(m.config.Network.SubnetCIDR) + if err != nil { + return nil, fmt.Errorf("derive gateway from subnet: %w", err) + } + } + + return newDefaultNetwork( + m.config.Network.BridgeName, + m.config.Network.SubnetCIDR, + gateway, + ), nil +} diff --git a/lib/network/default_network_linux_test.go b/lib/network/default_network_linux_test.go new file mode 100644 index 00000000..f96e271f --- /dev/null +++ b/lib/network/default_network_linux_test.go @@ -0,0 +1,90 @@ +//go:build linux + +package network + +import ( + "context" + "net" + "testing" + + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vishvananda/netlink" +) + +func TestLinuxEffectiveDefaultNetworkUsesBridgeConfig(t *testing.T) { + cfg := &config.Config{ + Network: config.NetworkConfig{ + BridgeName: "testbr0", + SubnetCIDR: "10.123.0.0/16", + SubnetGateway: "10.123.0.42", + }, + } + m := NewManager(paths.New(t.TempDir()), cfg, nil) + + effective, err := m.EffectiveDefaultNetwork() + require.NoError(t, err) + assert.Equal(t, "testbr0", effective.Bridge) + assert.Equal(t, "10.123.0.0/16", effective.Subnet) + assert.Equal(t, "10.123.0.42", effective.Gateway) +} + +func TestLinuxEffectiveDefaultNetworkDerivesGateway(t *testing.T) { + cfg := &config.Config{ + Network: config.NetworkConfig{ + BridgeName: "testbr0", + SubnetCIDR: "10.124.0.0/16", + }, + } + m := NewManager(paths.New(t.TempDir()), cfg, nil) + + effective, err := m.EffectiveDefaultNetwork() + require.NoError(t, err) + assert.Equal(t, "10.124.0.1", effective.Gateway) +} + +func TestSelectBridgeGatewayAddrPrefersConfiguredGateway(t *testing.T) { + addrs := []netlink.Addr{ + {IPNet: &net.IPNet{IP: net.ParseIP("10.123.0.2"), Mask: net.CIDRMask(16, 32)}}, + {IPNet: &net.IPNet{IP: net.ParseIP("10.123.0.1"), Mask: net.CIDRMask(16, 32)}}, + } + + selected := selectBridgeGatewayAddr(addrs, net.ParseIP("10.123.0.1")) + assert.Equal(t, "10.123.0.1", selected.IP.String()) +} + +func TestCanonicalSubnetCIDRUsesNetworkAddress(t *testing.T) { + assert.Equal(t, "10.123.0.0/16", canonicalSubnetCIDR(&net.IPNet{ + IP: net.ParseIP("10.123.0.42"), + Mask: net.CIDRMask(16, 32), + })) +} + +func TestBridgeAddressMatchesRequiresGatewayPrefix(t *testing.T) { + addrs := []netlink.Addr{ + {IPNet: &net.IPNet{IP: net.ParseIP("10.123.0.1"), Mask: net.CIDRMask(16, 32)}}, + } + + hasGateway, hasMatchingMask := bridgeAddressMatches(addrs, net.ParseIP("10.123.0.1"), net.CIDRMask(24, 32)) + assert.True(t, hasGateway) + assert.False(t, hasMatchingMask) + + hasGateway, hasMatchingMask = bridgeAddressMatches(addrs, net.ParseIP("10.123.0.1"), net.CIDRMask(16, 32)) + assert.True(t, hasGateway) + assert.True(t, hasMatchingMask) +} + +func TestGetDefaultNetworkPreservesLookupError(t *testing.T) { + cfg := &config.Config{ + Network: config.NetworkConfig{BridgeName: "hypeman-no-such-bridge"}, + } + m := NewManager(paths.New(t.TempDir()), cfg, nil).(*manager) + + _, err := m.getDefaultNetwork(context.Background()) + require.Error(t, err) + assert.ErrorIs(t, err, ErrNotFound) + assert.ErrorContains(t, err, "query default network state") + assert.ErrorContains(t, err, "look up bridge") +} diff --git a/lib/network/derive.go b/lib/network/derive.go index 0f43e35a..0ab980f3 100644 --- a/lib/network/derive.go +++ b/lib/network/derive.go @@ -39,22 +39,18 @@ func (m *manager) deriveAllocation(ctx context.Context, instanceID string) (*All return nil, nil } - // 3. Derive gateway/netmask from configured subnet. - // This avoids transient dependence on live bridge state when callers only need - // metadata-derived allocation details (e.g., immediately after instance create). - subnet := m.config.Network.SubnetCIDR - _, ipNet, err := net.ParseCIDR(subnet) + // 3. Derive gateway/netmask from the same platform-effective network used + // for new allocations. This remains available without querying transient live + // bridge state and prevents raw config from overriding VZ's NAT network. + defaultNetwork, err := m.EffectiveDefaultNetwork() + if err != nil { + return nil, fmt.Errorf("get effective default network: %w", err) + } + _, ipNet, err := net.ParseCIDR(defaultNetwork.Subnet) if err != nil { return nil, fmt.Errorf("parse subnet CIDR: %w", err) } netmask := fmt.Sprintf("%d.%d.%d.%d", ipNet.Mask[0], ipNet.Mask[1], ipNet.Mask[2], ipNet.Mask[3]) - gateway := m.config.Network.SubnetGateway - if gateway == "" { - gateway, err = DeriveGateway(subnet) - if err != nil { - return nil, fmt.Errorf("derive gateway from subnet: %w", err) - } - } // 4. Use stored metadata to derive allocation (works for all hypervisors) if meta.IP != "" && meta.MAC != "" { @@ -81,7 +77,7 @@ func (m *manager) deriveAllocation(ctx context.Context, instanceID string) (*All IP: meta.IP, MAC: meta.MAC, TAPDevice: tap, - Gateway: gateway, + Gateway: defaultNetwork.Gateway, Netmask: netmask, DNS: m.config.Network.DNSServer, State: state, diff --git a/lib/network/manager.go b/lib/network/manager.go index bc657117..bdfacad9 100644 --- a/lib/network/manager.go +++ b/lib/network/manager.go @@ -34,6 +34,10 @@ type Manager interface { GetAllocation(ctx context.Context, instanceID string) (*Allocation, error) ListAllocations(ctx context.Context) ([]Allocation, error) NameExists(ctx context.Context, name string, excludeInstanceID string) (bool, error) + // EffectiveDefaultNetwork returns the platform backend's default network. + // Before Initialize it returns the network the backend will create or attach; + // after Initialize it returns the backend state cached by the manager. + EffectiveDefaultNetwork() (*Network, error) // CleanupOrphanedTAPs removes TAP devices not associated with any preserved // instance. Pass minAge>0 to skip TAPs younger than that, which avoids racing @@ -92,40 +96,34 @@ func NewManager(p *paths.Paths, cfg *config.Config, meter metric.Meter) Manager func (m *manager) Initialize(ctx context.Context, runningInstanceIDs []string) error { log := logger.FromContext(ctx) - // Derive gateway from subnet if not explicitly configured - gateway := m.config.Network.SubnetGateway - if gateway == "" { - var err error - gateway, err = DeriveGateway(m.config.Network.SubnetCIDR) - if err != nil { - return fmt.Errorf("derive gateway from subnet: %w", err) - } + requestedNetwork, err := m.platformDefaultNetwork() + if err != nil { + return err } log.InfoContext(ctx, "initializing network manager", - "bridge", m.config.Network.BridgeName, - "subnet", m.config.Network.SubnetCIDR, - "gateway", gateway) + "bridge", requestedNetwork.Bridge, + "subnet", requestedNetwork.Subnet, + "gateway", requestedNetwork.Gateway) - // Check for subnet conflicts with existing host routes before creating bridge - if err := m.checkSubnetConflicts(ctx, m.config.Network.SubnetCIDR); err != nil { + // Check for subnet conflicts with existing host routes before creating bridge. + if err := m.checkSubnetConflicts(ctx, requestedNetwork.Subnet); err != nil { return err } - // Ensure default network bridge exists and iptables rules are configured - // createBridge is idempotent - handles both new and existing bridges - if err := m.createBridge(ctx, m.config.Network.BridgeName, gateway, m.config.Network.SubnetCIDR); err != nil { + // Ensure the platform backend is initialized. On Linux this creates the + // configured bridge; on Darwin VZ supplies its own NAT network. + if err := m.createBridge(ctx, requestedNetwork.Bridge, requestedNetwork.Gateway, requestedNetwork.Subnet); err != nil { return fmt.Errorf("setup default network: %w", err) } - m.setDefaultNetwork(&Network{ - Name: "default", - Subnet: m.config.Network.SubnetCIDR, - Gateway: gateway, - Bridge: m.config.Network.BridgeName, - // Per-TAP port isolation is the default network policy used by createTAPDevice. - Isolated: true, - Default: true, - }) + + // The backend state is authoritative once initialization has completed. In + // particular, VZ's NAT subnet must win over Linux-oriented config defaults. + effectiveNetwork, err := m.getDefaultNetwork(ctx) + if err != nil { + return fmt.Errorf("get effective default network: %w", err) + } + m.setDefaultNetwork(effectiveNetwork) // Cleanup orphaned TAP devices from previous runs (crashes, power loss, etc.). // Startup runs before any concurrent CreateAllocation can be in flight, so no @@ -163,23 +161,41 @@ func (m *manager) setDefaultNetwork(network *Network) { m.defaultNetwork = cloneNetwork(network) } -// getDefaultNetwork gets the default network details from kernel state +func newDefaultNetwork(bridge, subnet, gateway string) *Network { + return &Network{ + Name: "default", + Subnet: subnet, + Gateway: gateway, + Bridge: bridge, + Isolated: true, + Default: true, + } +} + +// EffectiveDefaultNetwork returns one platform-authoritative network identity. +// The pre-initialization value is useful to dependency providers that need the +// guest-visible gateway before the manager's startup initialization runs. +func (m *manager) EffectiveDefaultNetwork() (*Network, error) { + if network := m.cachedDefaultNetwork(); network != nil { + return network, nil + } + return m.platformDefaultNetwork() +} + +// getDefaultNetwork gets the default network details from backend state. func (m *manager) getDefaultNetwork(ctx context.Context) (*Network, error) { - // Query from kernel state, err := m.queryNetworkState(m.config.Network.BridgeName) if err != nil { - return nil, ErrNotFound + return nil, fmt.Errorf("query default network state: %w", err) } - return &Network{ - Name: "default", - Subnet: state.Subnet, - Gateway: state.Gateway, - Bridge: m.config.Network.BridgeName, - Isolated: true, - Default: true, - CreatedAt: time.Time{}, // Unknown for default - }, nil + bridge := state.Bridge + if bridge == "" { + bridge = m.config.Network.BridgeName + } + network := newDefaultNetwork(bridge, state.Subnet, state.Gateway) + network.CreatedAt = time.Time{} // Unknown for default + return network, nil } // SetupHTB initializes HTB qdisc on the bridge for upload fair sharing. diff --git a/lib/network/manager_darwin_test.go b/lib/network/manager_darwin_test.go new file mode 100644 index 00000000..b9811af5 --- /dev/null +++ b/lib/network/manager_darwin_test.go @@ -0,0 +1,81 @@ +//go:build darwin + +package network + +import ( + "context" + "encoding/json" + "net" + "os" + "testing" + + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newDarwinTestManager(t *testing.T) *manager { + t.Helper() + cfg := &config.Config{ + Network: config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + SubnetGateway: "10.100.0.99", + DNSServer: "1.1.1.1", + }, + } + return NewManager(paths.New(t.TempDir()), cfg, nil).(*manager) +} + +func TestDarwinEffectiveDefaultNetworkIgnoresLinuxConfig(t *testing.T) { + m := newDarwinTestManager(t) + + beforeInitialize, err := m.EffectiveDefaultNetwork() + require.NoError(t, err) + assert.Equal(t, vzNATBridge, beforeInitialize.Bridge) + assert.Equal(t, vzNATSubnet, beforeInitialize.Subnet) + assert.Equal(t, vzNATGateway, beforeInitialize.Gateway) + + require.NoError(t, m.Initialize(context.Background(), nil)) + afterInitialize, err := m.EffectiveDefaultNetwork() + require.NoError(t, err) + assert.Equal(t, beforeInitialize, afterInitialize) +} + +func TestDarwinAllocationAndDerivationUseVZNATNetwork(t *testing.T) { + ctx := context.Background() + m := newDarwinTestManager(t) + require.NoError(t, m.Initialize(ctx, nil)) + + const instanceID = "darwin-network-regression" + networkConfig, err := m.CreateAllocation(ctx, AllocateRequest{ + InstanceID: instanceID, + InstanceName: "darwin-network-regression", + }) + require.NoError(t, err) + assert.Equal(t, vzNATGateway, networkConfig.Gateway) + assert.Equal(t, "255.255.255.0", networkConfig.Netmask) + assert.Equal(t, "1.1.1.1", networkConfig.DNS) + _, vzSubnet, err := net.ParseCIDR(vzNATSubnet) + require.NoError(t, err) + assert.True(t, vzSubnet.Contains(net.ParseIP(networkConfig.IP))) + + metadata, err := json.Marshal(instanceMetadata{ + Name: "darwin-network-regression", + NetworkEnabled: true, + HypervisorType: "vz", + IP: networkConfig.IP, + MAC: networkConfig.MAC, + }) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(m.paths.InstanceDir(instanceID), 0o755)) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata(instanceID), metadata, 0o644)) + + allocation, err := m.GetAllocation(ctx, instanceID) + require.NoError(t, err) + require.NotNil(t, allocation) + assert.Equal(t, vzNATGateway, allocation.Gateway) + assert.Equal(t, "255.255.255.0", allocation.Netmask) + assert.Equal(t, networkConfig.IP, allocation.IP) +} diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 942e7095..1b52e3e4 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -391,7 +391,7 @@ func ProvideBuilderManager(p *paths.Paths, cfg *config.Config, instanceManager i } // ProvideBuildManager provides the build manager -func ProvideBuildManager(p *paths.Paths, cfg *config.Config, instanceManager instances.Manager, volumeManager volumes.Manager, builderManager builders.Manager, imageManager images.Manager, log *slog.Logger) (builds.Manager, error) { +func ProvideBuildManager(p *paths.Paths, cfg *config.Config, instanceManager instances.Manager, volumeManager volumes.Manager, builderManager builders.Manager, imageManager images.Manager, networkManager network.Manager, log *slog.Logger) (builds.Manager, error) { // Read CA cert file if specified var registryCACert string if cfg.Registry.CACertFile != "" { @@ -403,25 +403,12 @@ func ProvideBuildManager(p *paths.Paths, cfg *config.Config, instanceManager ins log.Info("registry CA certificate loaded", "file", cfg.Registry.CACertFile) } - // Rewrite localhost in RegistryURL to the subnet gateway IP so builder VMs - // (which run in their own network namespace) can reach the host registry. - // Inside a VM, "localhost" refers to the VM itself, not the host. - registryURL := cfg.Registry.URL - if registryURL == "" { - registryURL = "localhost:4973" + registryURL, err := builderRegistryURL(cfg.Registry.URL, networkManager) + if err != nil { + return nil, err } - if strings.HasPrefix(registryURL, "localhost:") || strings.HasPrefix(registryURL, "127.0.0.1:") { - gateway := cfg.Network.SubnetGateway - if gateway == "" { - var err error - gateway, err = network.DeriveGateway(cfg.Network.SubnetCIDR) - if err != nil { - return nil, fmt.Errorf("derive gateway for registry URL rewrite: %w", err) - } - } - port := strings.SplitN(registryURL, ":", 2)[1] - registryURL = gateway + ":" + port - log.Info("rewrote registry URL for builder VMs", "original", cfg.Registry.URL, "rewritten", registryURL) + if registryURL != cfg.Registry.URL { + log.Info("resolved registry URL for builder VMs", "original", cfg.Registry.URL, "resolved", registryURL) } buildConfig := builds.Config{ @@ -455,3 +442,22 @@ func ProvideBuildManager(p *paths.Paths, cfg *config.Config, instanceManager ins return buildManager, nil } + +// builderRegistryURL rewrites loopback registry addresses to the host gateway +// visible to guests. The network manager owns platform-specific gateway selection. +func builderRegistryURL(configuredURL string, networkManager network.Manager) (string, error) { + registryURL := configuredURL + if registryURL == "" { + registryURL = "localhost:4973" + } + if !strings.HasPrefix(registryURL, "localhost:") && !strings.HasPrefix(registryURL, "127.0.0.1:") { + return registryURL, nil + } + + defaultNetwork, err := networkManager.EffectiveDefaultNetwork() + if err != nil { + return "", fmt.Errorf("get effective default network for registry URL rewrite: %w", err) + } + port := strings.SplitN(registryURL, ":", 2)[1] + return defaultNetwork.Gateway + ":" + port, nil +} diff --git a/lib/providers/providers_darwin_test.go b/lib/providers/providers_darwin_test.go new file mode 100644 index 00000000..1d621e92 --- /dev/null +++ b/lib/providers/providers_darwin_test.go @@ -0,0 +1,34 @@ +//go:build darwin + +package providers + +import ( + "testing" + + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuilderRegistryURLUsesDarwinEffectiveGateway(t *testing.T) { + cfg := &config.Config{ + Network: config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + SubnetGateway: "10.100.0.1", + }, + } + networkManager := network.NewManager(paths.New(t.TempDir()), cfg, nil) + + for _, registryURL := range []string{"", "localhost:4973", "127.0.0.1:5000"} { + resolved, err := builderRegistryURL(registryURL, networkManager) + require.NoError(t, err) + if registryURL == "127.0.0.1:5000" { + assert.Equal(t, "192.168.64.1:5000", resolved) + } else { + assert.Equal(t, "192.168.64.1:4973", resolved) + } + } +} diff --git a/scripts/e2e-install-test.sh b/scripts/e2e-install-test.sh index 1e3b28d8..368a5549 100755 --- a/scripts/e2e-install-test.sh +++ b/scripts/e2e-install-test.sh @@ -192,12 +192,89 @@ OUTPUT=$($HYPEMAN_CMD exec "$E2E_VM_NAME" -- echo "hello from e2e") || fail "hyp echo "$OUTPUT" | grep -q "hello from e2e" || fail "hypeman exec output mismatch: $OUTPUT" pass "hypeman exec works" +# Exec uses vsock and can succeed when guest networking is broken. Exercise DNS +# and an outbound TCP connection so the macOS VZ NAT path is covered end to end. +EGRESS_OK=false +for i in $(seq 1 3); do + if $HYPEMAN_CMD exec "$E2E_VM_NAME" -- wget -q -T 10 -O /dev/null http://example.com; then + EGRESS_OK=true + break + fi + warn "guest egress attempt ${i}/3 failed" + sleep 2 +done +[ "$EGRESS_OK" = true ] || fail "guest TCP egress failed" +pass "guest TCP egress works" + $HYPEMAN_CMD stop "$E2E_VM_NAME" || fail "hypeman stop failed" pass "hypeman stop works" $HYPEMAN_CMD rm "$E2E_VM_NAME" || fail "hypeman rm failed" pass "hypeman rm works" +# Build a real image through the installed CLI. This exercises provider wiring, +# the VZ builder VM, guest-to-host registry access, and the resulting image. +E2E_BUILD_VM_NAME="e2e-build-test-vm" +BUILD_CONTEXT=$(mktemp -d) +BUILD_OUTPUT_FILE=$(mktemp) +trap 'rm -rf "${BUILD_CONTEXT:-}" "${BUILD_OUTPUT_FILE:-}"' EXIT +cat > "$BUILD_CONTEXT/Dockerfile" <<'EOF' +FROM alpine:3.20 +RUN printf 'hypeman-build-e2e\n' > /build-marker +CMD ["sh", "-c", "sleep 600"] +EOF + +BUILD_OK=false +for i in $(seq 1 30); do + if $HYPEMAN_CMD build --file Dockerfile --timeout 600 \ + --image-name e2e/build-smoke:latest "$BUILD_CONTEXT" >"$BUILD_OUTPUT_FILE" 2>&1; then + BUILD_OK=true + break + fi + if ! grep -q "builder image is being prepared" "$BUILD_OUTPUT_FILE"; then + cat "$BUILD_OUTPUT_FILE" + FAILED_BUILD_ID=$(sed -n 's/^Build started: //p' "$BUILD_OUTPUT_FILE" | tail -1) + if [ -n "$FAILED_BUILD_ID" ]; then + $HYPEMAN_CMD --format json build get "$FAILED_BUILD_ID" || true + fi + for LOG_FILE in "$HOME/Library/Application Support/hypeman/logs/hypeman.log" /var/lib/hypeman/logs/hypeman.log; do + if [ -f "$LOG_FILE" ]; then + tail -200 "$LOG_FILE" + fi + done + fail "hypeman build failed" + fi + warn "builder image is still being prepared (${i}/30)" + sleep 5 +done +[ "$BUILD_OK" = true ] || { cat "$BUILD_OUTPUT_FILE"; fail "hypeman build did not become ready"; } +cat "$BUILD_OUTPUT_FILE" +BUILD_ID=$(sed -n 's/^Build started: //p' "$BUILD_OUTPUT_FILE" | tail -1) +[ -n "$BUILD_ID" ] || fail "hypeman build output did not include a build ID" +BUILD_IMAGE=$($HYPEMAN_CMD --format json --transform image_ref build get "$BUILD_ID") || fail "hypeman build get failed" +BUILD_IMAGE=${BUILD_IMAGE#\"} +BUILD_IMAGE=${BUILD_IMAGE%\"} +[ -n "$BUILD_IMAGE" ] || fail "completed build did not include image_ref" +pass "hypeman build works" + +$HYPEMAN_CMD run --name "$E2E_BUILD_VM_NAME" "$BUILD_IMAGE" || fail "running built image failed" +BUILD_VM_READY=false +for i in $(seq 1 30); do + if OUTPUT=$($HYPEMAN_CMD exec "$E2E_BUILD_VM_NAME" -- cat /build-marker 2>/dev/null); then + if echo "$OUTPUT" | grep -q "hypeman-build-e2e"; then + BUILD_VM_READY=true + break + fi + fi + sleep 2 +done +[ "$BUILD_VM_READY" = true ] || fail "built image did not become ready with expected marker" +pass "image produced by hypeman build runs" + +$HYPEMAN_CMD stop "$E2E_BUILD_VM_NAME" || fail "stopping built image failed" +$HYPEMAN_CMD rm "$E2E_BUILD_VM_NAME" || fail "removing built image instance failed" +rm -rf "$BUILD_CONTEXT" "$BUILD_OUTPUT_FILE" + # ============================================================================= # Phase 5: Cleanup # ============================================================================= diff --git a/scripts/install.sh b/scripts/install.sh index 3350a25d..67405d46 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -573,8 +573,6 @@ if [ "$OS" = "darwin" ]; then PLIST - info "Loading ${SERVICE_NAME} service..." - launchctl load "$PLIST_PATH" else # Linux: systemd info "Installing systemd service..." @@ -673,6 +671,11 @@ if [ "$OS" = "darwin" ]; then fi fi +if [ "$OS" = "darwin" ]; then + info "Loading ${SERVICE_NAME} service..." + launchctl load "$PLIST_PATH" +fi + # ============================================================================= # Install Hypeman CLI # =============================================================================