Fix PDS disk boot failures (MIR-788)

appconfig/appconfig.go

@@ -1,6 +1,7 @@
 package appconfig
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -97,6 +98,7 @@ func LoadAppConfig() (*AppConfig, error) {
 
 			var ac AppConfig
 			dec := toml.NewDecoder(fi)
+			dec.DisallowUnknownFields()
 			err = dec.Decode(&ac)
 			if err != nil {
 				return nil, err
@@ -124,6 +126,7 @@ func LoadAppConfigUnder(dir string) (*AppConfig, error) {
 
 		var ac AppConfig
 		dec := toml.NewDecoder(fi)
+		dec.DisallowUnknownFields()
 		err = dec.Decode(&ac)
 		if err != nil {
 			return nil, err
@@ -142,7 +145,9 @@ func LoadAppConfigUnder(dir string) (*AppConfig, error) {
 
 func Parse(data []byte) (*AppConfig, error) {
 	var ac AppConfig
-	err := toml.Unmarshal(data, &ac)
+	dec := toml.NewDecoder(bytes.NewReader(data))
+	dec.DisallowUnknownFields()
+	err := dec.Decode(&ac)
 	if err != nil {
 		return nil, err
 	}

appconfig/appconfig_test.go

@@ -1136,3 +1136,65 @@ node_port = 443
 		require.Len(t, ac.Services["web"].Ports, 2)
 	})
 }
+
+func TestRejectUnknownFields(t *testing.T) {
+	t.Run("unknown top-level field", func(t *testing.T) {
+		config := `
+name = "test-app"
+unknown_field = "value"
+`
+		_, err := Parse([]byte(config))
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "strict mode")
+	})
+
+	t.Run("size instead of size_gb in disk config", func(t *testing.T) {
+		config := `
+name = "test-app"
+
+[services.database.concurrency]
+mode = "fixed"
+num_instances = 1
+
+[[services.database.disks]]
+name = "data"
+mount_path = "/data"
+size = 20
+`
+		_, err := Parse([]byte(config))
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "strict mode")
+	})
+
+	t.Run("unknown field in service config", func(t *testing.T) {
+		config := `
+name = "test-app"
+
+[services.web]
+command = "server"
+bogus = true
+`
+		_, err := Parse([]byte(config))
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "strict mode")
+	})
+
+	t.Run("valid config still works", func(t *testing.T) {
+		config := `
+name = "test-app"
+
+[services.database.concurrency]
+mode = "fixed"
+num_instances = 1
+
+[[services.database.disks]]
+name = "data"
+mount_path = "/data"
+size_gb = 20
+`
+		ac, err := Parse([]byte(config))
+		require.NoError(t, err)
+		require.NotNil(t, ac)
+		assert.Equal(t, 20, ac.Services["database"].Disks[0].SizeGB)
+	})
+}

controllers/sandbox/sandbox.go

@@ -838,9 +838,19 @@ func (c *SandboxController) Create(ctx context.Context, co *compute.Sandbox, met
 	}
 }
 
-func (c *SandboxController) createSandbox(ctx context.Context, co *compute.Sandbox, meta *entity.Meta, recreate bool) error {
+func (c *SandboxController) createSandbox(ctx context.Context, co *compute.Sandbox, meta *entity.Meta, recreate bool) (err error) {
 	c.Log.Debug("creating sandbox", "id", co.ID)
 
+	// Catch-all: any error during sandbox creation marks it DEAD so the pool
+	// controller's crash-backoff logic kicks in instead of retrying forever.
+	defer func() {
+		if err != nil {
+			c.Log.Error("sandbox boot failed, marking DEAD", "id", co.ID, "err", err)
+			co.Status = compute.DEAD
+			meta.Update(co.Encode())
+		}
+	}()
+
 	ctx = namespaces.WithNamespace(ctx, c.Namespace)
 
 	ep, err := c.allocateNetwork(ctx, co)
@@ -894,7 +904,7 @@ func (c *SandboxController) createSandbox(ctx context.Context, co *compute.Sandb
 
 	defer func() {
 		if err != nil {
-			c.Log.Error("failed to create sandbox, cleaning up", "id", co.ID, "err", err)
+			c.Log.Error("failed to create sandbox, cleaning up container resources", "id", co.ID, "err", err)
 
 			// Be sure we have at least 60 seconds to do this action.
 			ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
@@ -908,11 +918,6 @@ func (c *SandboxController) createSandbox(ctx context.Context, co *compute.Sandb
 
 			// Clean up the pause container using the common cleanup function
 			c.cleanupContainer(ctx, container)
-
-			// Update sandbox status to DEAD in entity store
-			co.Status = compute.DEAD
-			meta.Update(co.Encode())
-			c.Log.Info("marked sandbox as DEAD due to boot failure", "id", co.ID)
 		}
 	}()
 

controllers/sandbox/volume.go

@@ -132,12 +132,6 @@ func (c *SandboxController) configureMirenVolume(ctx context.Context, sb *comput
 		leaseTimeout = duration
 	}
 
-	// Look up or create Disk entity using instance-specific name
-	diskID, err := c.ensureDisk(ctx, actualDiskName, sizeGB, filesystem)
-	if err != nil {
-		return "", fmt.Errorf("failed to ensure disk exists: %w", err)
-	}
-
 	// Resolve version to app ID if set
 	var appID entity.Id
 	if sb.Spec.Version != "" {
@@ -153,6 +147,12 @@ func (c *SandboxController) configureMirenVolume(ctx context.Context, sb *comput
 		}
 	}
 
+	// Look up or create Disk entity using instance-specific name
+	diskID, err := c.ensureDisk(ctx, actualDiskName, sizeGB, filesystem, appID)
+	if err != nil {
+		return "", fmt.Errorf("failed to ensure disk exists: %w", err)
+	}
+
 	// Acquire a lease for this disk on this node (creates new or takes over existing)
 	nodeID := entity.Id("node/" + c.NodeId)
 	leaseID, err := c.acquireDiskLease(ctx, diskID, nodeID, sb.ID, appID, volume.MountPath, readOnly)
@@ -174,7 +174,7 @@ func (c *SandboxController) configureMirenVolume(ctx context.Context, sb *comput
 	return diskMountPath, nil
 }
 
-func (c *SandboxController) ensureDisk(ctx context.Context, diskName string, sizeGB int64, filesystem string) (entity.Id, error) {
+func (c *SandboxController) ensureDisk(ctx context.Context, diskName string, sizeGB int64, filesystem string, appID entity.Id) (entity.Id, error) {
 	// Search for existing disk by name using the name index
 	listResp, err := c.EAC.List(ctx, entity.String(storage.DiskNameId, diskName))
 	if err != nil {
@@ -222,6 +222,7 @@ func (c *SandboxController) ensureDisk(ctx context.Context, diskName string, siz
 		SizeGb:     sizeGB,
 		Filesystem: fs,
 		Status:     storage.PROVISIONING,
+		CreatedBy:  appID,
 	}
 
 	name := idgen.GenNS("disk")

servers/build/build.go

@@ -27,6 +27,7 @@ import (
 	"miren.dev/runtime/api/entityserver"
 	"miren.dev/runtime/api/entityserver/entityserver_v1alpha"
 	"miren.dev/runtime/api/ingress"
+	storage "miren.dev/runtime/api/storage/storage_v1alpha"
 	"miren.dev/runtime/appconfig"
 	"miren.dev/runtime/components/buildkit"
 	"miren.dev/runtime/components/netresolve"
@@ -294,6 +295,28 @@ func validateNodePorts(ctx context.Context, eac *entityserver_v1alpha.EntityAcce
 	return nil
 }
 
+// validateDiskConfigs checks that disks referenced with size_gb=0 already exist
+// in the entity store. This catches missing disks at deploy time rather than
+// letting sandboxes fail at runtime with retry loops.
+func validateDiskConfigs(ctx context.Context, eac *entityserver_v1alpha.EntityAccessClient, spec core_v1alpha.ConfigSpec) error {
+	for _, svc := range spec.Services {
+		for _, disk := range svc.Disks {
+			if disk.SizeGb > 0 || disk.Name == "" {
+				continue
+			}
+			// size_gb == 0 means we expect the disk to already exist
+			listResp, err := eac.List(ctx, entity.String(storage.DiskNameId, disk.Name))
+			if err != nil {
+				return fmt.Errorf("failed to query disk %q: %w", disk.Name, err)
+			}
+			if len(listResp.Values()) == 0 {
+				return fmt.Errorf("disk %q does not exist; set size_gb to auto-create it", disk.Name)
+			}
+		}
+	}
+	return nil
+}
+
 // buildServicesConfig collects services from app config and procfile,
 // resolves defaults, and returns the final service configurations.
 // This is the core logic for determining which services exist in an app_version
@@ -1124,6 +1147,11 @@ func (b *Builder) BuildFromTar(ctx context.Context, state *build_v1alpha.Builder
 		return err
 	}
 
+	if err := validateDiskConfigs(ctx, b.ec.EAC(), configSpec); err != nil {
+		b.sendErrorStatus(ctx, status, "Deploy failed: %v", err)
+		return err
+	}
+
 	if len(args.EnvVars()) > 0 {
 		b.Log.Info("applied CLI env vars", "count", len(args.EnvVars()))
 	}

servers/build/build_test.go

@@ -11,6 +11,7 @@ import (
 	"miren.dev/runtime/api/build/build_v1alpha"
 	"miren.dev/runtime/api/compute/compute_v1alpha"
 	"miren.dev/runtime/api/core/core_v1alpha"
+	storage "miren.dev/runtime/api/storage/storage_v1alpha"
 	"miren.dev/runtime/appconfig"
 	"miren.dev/runtime/pkg/entity"
 	"miren.dev/runtime/pkg/entity/testutils"
@@ -2289,3 +2290,77 @@ func TestValidateNodePortsScaledDownPoolIgnored(t *testing.T) {
 	err = validateNodePorts(ctx, server.EAC, appBID, spec)
 	assert.NoError(t, err)
 }
+
+func TestValidateDiskConfigsMissingDisk(t *testing.T) {
+	ctx := context.Background()
+	server, cleanup := testutils.NewInMemEntityServer(t)
+	defer cleanup()
+
+	// Deploy with size_gb=0 and no existing disk — should fail
+	spec := core_v1alpha.ConfigSpec{
+		Services: []core_v1alpha.ConfigSpecServices{{
+			Name: "db",
+			Disks: []core_v1alpha.ConfigSpecServicesDisks{{
+				Name:      "data",
+				MountPath: "/data",
+				SizeGb:    0,
+			}},
+		}},
+	}
+
+	err := validateDiskConfigs(ctx, server.EAC, spec)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), `disk "data" does not exist`)
+	assert.Contains(t, err.Error(), "size_gb")
+}
+
+func TestValidateDiskConfigsExistingDisk(t *testing.T) {
+	ctx := context.Background()
+	server, cleanup := testutils.NewInMemEntityServer(t)
+	defer cleanup()
+
+	// Create a disk entity so the lookup succeeds
+	disk := &storage.Disk{
+		Name:   "data",
+		SizeGb: 20,
+		Status: storage.PROVISIONED,
+	}
+	_, err := server.Client.Create(ctx, "data-disk", disk)
+	require.NoError(t, err)
+
+	// Deploy with size_gb=0 referencing an existing disk — should succeed
+	spec := core_v1alpha.ConfigSpec{
+		Services: []core_v1alpha.ConfigSpecServices{{
+			Name: "db",
+			Disks: []core_v1alpha.ConfigSpecServicesDisks{{
+				Name:      "data",
+				MountPath: "/data",
+				SizeGb:    0,
+			}},
+		}},
+	}
+
+	err = validateDiskConfigs(ctx, server.EAC, spec)
+	assert.NoError(t, err)
+}
+
+func TestValidateDiskConfigsAutoCreate(t *testing.T) {
+	ctx := context.Background()
+	server, cleanup := testutils.NewInMemEntityServer(t)
+	defer cleanup()
+
+	// Deploy with size_gb > 0 — should always succeed (will auto-create)
+	spec := core_v1alpha.ConfigSpec{
+		Services: []core_v1alpha.ConfigSpecServices{{
+			Name: "db",
+			Disks: []core_v1alpha.ConfigSpecServicesDisks{{
+				Name:      "new-disk",
+				MountPath: "/data",
+				SizeGb:    20,
+			}},
+		}},
+	}
+
+	err := validateDiskConfigs(ctx, server.EAC, spec)
+	assert.NoError(t, err)
+}