Retry builder image preparation at startup

Author: cursoragent

lib/builds/manager.go

@@ -34,6 +34,10 @@ const (
 	// of a build; releaseBuildRetryDelay spaces the attempts.
 	releaseBuildMaxAttempts = 5
 	releaseBuildRetryDelay  = time.Second
+
+	// builderImageRetryDelay controls how often startup retries builder image
+	// preparation after a transient failure.
+	builderImageRetryDelay = time.Second
 )
 
 //go:embed images/generic/Dockerfile
@@ -205,11 +209,19 @@ func NewManager(
 // Start starts the build manager's background services
 func (m *manager) Start(ctx context.Context) error {
 	go func() {
-		m.ensureBuilderImage(ctx)
-		// Recover pending builds only after the builder image is ready,
-		// otherwise recovered builds fail with "builder image is being prepared".
-		if m.ReadyForBuilds() {
-			m.RecoverPendingBuilds()
+		for {
+			m.ensureBuilderImage(ctx)
+			// Recover pending builds only after the builder image is ready,
+			// otherwise recovered builds fail with "builder image is being prepared".
+			if m.ReadyForBuilds() {
+				m.RecoverPendingBuilds()
+				return
+			}
+			select {
+			case <-ctx.Done():
+				return
+			case <-time.After(builderImageRetryDelay):
+			}
 		}
 	}()
 	m.logger.Info("build manager started")

lib/builds/manager_test.go

@@ -309,9 +309,10 @@ func (m *mockSecretProvider) GetSecrets(ctx context.Context, secretIDs []string)
 
 // mockImageManager implements images.Manager for testing
 type mockImageManager struct {
-	mu          sync.RWMutex
-	images      map[string]*images.Image
-	getImageErr error
+	mu              sync.RWMutex
+	images          map[string]*images.Image
+	createImageFunc func(ctx context.Context, req images.CreateImageRequest) (*images.Image, error)
+	getImageErr     error
 }
 
 func newMockImageManager() *mockImageManager {
@@ -329,6 +330,9 @@ func (m *mockImageManager) ListImages(ctx context.Context) ([]images.Image, erro
 }
 
 func (m *mockImageManager) CreateImage(ctx context.Context, req images.CreateImageRequest) (*images.Image, error) {
+	if m.createImageFunc != nil {
+		return m.createImageFunc(ctx, req)
+	}
 	img := &images.Image{
 		Name:   req.Name,
 		Status: images.StatusPending,
@@ -898,6 +902,34 @@ func TestStart(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestStart_RetriesBuilderPreparationAfterFailure(t *testing.T) {
+	mgr, _, _, imageMgr, tempDir := setupTestManagerWithImageMgr(t)
+	defer os.RemoveAll(tempDir)
+	mgr.builderReady.Store(false)
+
+	attempts := 0
+	imageMgr.createImageFunc = func(ctx context.Context, req images.CreateImageRequest) (*images.Image, error) {
+		attempts++
+		if attempts == 1 {
+			return nil, fmt.Errorf("transient pull failure")
+		}
+		imageMgr.mu.Lock()
+		defer imageMgr.mu.Unlock()
+		img := &images.Image{
+			Name:   req.Name,
+			Status: images.StatusReady,
+		}
+		imageMgr.images[req.Name] = img
+		return img, nil
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	require.NoError(t, mgr.Start(ctx))
+	require.Eventually(t, mgr.ReadyForBuilds, 5*time.Second, 50*time.Millisecond)
+}
+
 func TestCreateBuild_MultipleConcurrent(t *testing.T) {
 	mgr, _, _, tempDir := setupTestManager(t)
 	defer os.RemoveAll(tempDir)