Optimize deploy upload with delta file transfer

api/build/rpc.yml

@@ -125,6 +125,42 @@ types:
         index: 2
         doc: Whether the value should be masked in logs and output
 
+  - type: FileManifestEntry
+    doc: A file entry in the upload manifest with its hash for delta comparison
+    fields:
+      - name: path
+        type: string
+        index: 0
+        doc: Relative file path
+      - name: hash
+        type: string
+        index: 1
+        doc: SHA-256 hash of the file contents
+      - name: size
+        type: int64
+        index: 2
+        doc: File size in bytes
+      - name: mode
+        type: int32
+        index: 3
+        doc: File permission mode
+
+  - type: PrepareUploadResult
+    doc: Result of preparing a delta upload session
+    fields:
+      - name: session_id
+        type: string
+        index: 0
+        doc: Session ID for the subsequent buildFromPrepared call
+      - name: needed_paths
+        type: '[]string'
+        index: 1
+        doc: File paths that need to be uploaded (not found in cache)
+      - name: cached_count
+        type: int32
+        index: 2
+        doc: Number of files reused from the previous deploy cache
+
 interfaces:
   - name: Stream
     methods:
@@ -165,3 +201,33 @@ interfaces:
           - name: result
             type: '*AnalysisResult'
 
+      - name: prepareUpload
+        doc: Prepare a delta upload session by comparing a file manifest against cached source code
+        parameters:
+          - name: application
+            type: string
+          - name: manifest
+            type: '[]*FileManifestEntry'
+        results:
+          - name: result
+            type: '*PrepareUploadResult'
+
+      - name: buildFromPrepared
+        doc: Build from a prepared upload session with only the changed files
+        parameters:
+          - name: session_id
+            type: string
+          - name: tardata
+            type: stream.RecvStream[[]byte]
+          - name: status
+            type: stream.SendStream[*Status]
+          - name: envVars
+            type: '[]*EnvironmentVariable'
+            doc: Environment variables to set on the app (optional, applied with source=manual)
+        results:
+          - name: version
+            type: string
+          - name: access_info
+            type: '*AccessInfo'
+            doc: Information about how to access the deployed app
+

cli/commands/deploy.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"strings"
 	"sync"
@@ -453,10 +454,61 @@ func Deploy(ctx *Context, opts struct {
 	// Update phase to building
 	updateDeploymentPhase("building")
 
-	// Start upload span covering tar creation + BuildFromTar
+	// Start upload span covering tar creation + build
 	_, uploadSpan := deployTracer.Start(buildCtx, "deploy.upload")
 
-	r, err := tarx.MakeTar(dir, includePatterns)
+	// Try optimized delta upload: compute manifest, ask server what's cached
+	var (
+		sessionID    string
+		useOptimized bool
+		totalFiles   int
+		cachedFiles  int32
+		neededPaths  map[string]bool
+	)
+
+	manifest, manifestErr := tarx.ComputeManifest(dir, includePatterns)
+	if manifestErr == nil {
+		totalFiles = len(manifest)
+
+		// Convert to RPC manifest entries
+		rpcManifest := make([]*build_v1alpha.FileManifestEntry, len(manifest))
+		for i, m := range manifest {
+			entry := &build_v1alpha.FileManifestEntry{}
+			entry.SetPath(m.Path)
+			entry.SetHash(m.Hash)
+			entry.SetSize(m.Size)
+			entry.SetMode(m.Mode)
+			rpcManifest[i] = entry
+		}
+
+		prepResult, prepErr := bc.PrepareUpload(buildCtx, name, rpcManifest)
+		if prepErr == nil && prepResult.Result() != nil {
+			result := prepResult.Result()
+			sessionID = result.SessionId()
+			cachedFiles = result.CachedCount()
+			useOptimized = true
+
+			if result.HasNeededPaths() && result.NeededPaths() != nil {
+				neededPaths = make(map[string]bool)
+				for _, p := range *result.NeededPaths() {
+					neededPaths[p] = true
+				}
+			}
+		} else {
+			ctx.Log.Debug("prepareUpload unavailable, falling back to full upload", "error", prepErr)
+		}
+	} else {
+		ctx.Log.Debug("manifest computation failed, falling back to full upload", "error", manifestErr)
+	}
+
+	var r io.ReadCloser
+	if useOptimized && len(neededPaths) > 0 {
+		r, err = tarx.MakeFilteredTar(dir, includePatterns, neededPaths)
+	} else if useOptimized {
+		r = tarx.MakeEmptyTar()
+	} else {
+		r, err = tarx.MakeTar(dir, includePatterns)
+	}
 	if err != nil {
 		uploadSpan.RecordError(err)
 		uploadSpan.SetStatus(codes.Error, err.Error())
@@ -467,16 +519,35 @@ func Deploy(ctx *Context, opts struct {
 
 	defer r.Close()
 
+	// buildCall wraps either BuildFromTar or BuildFromPrepared
+	type buildResults interface {
+		Version() string
+		HasAccessInfo() bool
+		AccessInfo() *build_v1alpha.AccessInfo
+	}
+	buildCall := func(callCtx context.Context, tarReader io.ReadCloser, cb stream.SendStream[*build_v1alpha.Status]) (buildResults, error) {
+		if useOptimized {
+			tarStream := stream.ServeReader(callCtx, tarReader)
+			return bc.BuildFromPrepared(callCtx, sessionID, tarStream, cb, envVars)
+		}
+		return bc.BuildFromTar(callCtx, name, stream.ServeReader(callCtx, tarReader), cb, envVars)
+	}
+
 	var (
 		cb      stream.SendStream[*build_v1alpha.Status]
-		results *build_v1alpha.BuilderClientBuildFromTarResults
+		results buildResults
 	)
 
 	// Detect if we have a TTY - if not, force explain mode
 	isTTY := term.IsTerminal(int(os.Stdout.Fd()))
 	useExplainMode := opts.Explain || !isTTY
 
 	if useExplainMode {
+		if useOptimized && cachedFiles > 0 {
+			faintStyle := lipgloss.NewStyle().Faint(true)
+			ctx.Printf("  %s\n", faintStyle.Render(fmt.Sprintf("Reused %d/%d files from previous deploy, uploading %d", cachedFiles, totalFiles, len(neededPaths))))
+		}
+
 		// In explain mode, write to stderr
 		pw, err := progresswriter.NewPrinter(ctx, os.Stderr, opts.ExplainFormat)
 		if err != nil {
@@ -525,7 +596,7 @@ func Deploy(ctx *Context, opts struct {
 
 		cb = createBuildStatusCallback(buildCtx, nil, nil, &buildErrors, nil, progressHandler)
 
-		results, err = bc.BuildFromTar(buildCtx, name, stream.ServeReader(buildCtx, r), cb, envVars)
+		results, err = buildCall(buildCtx, r, cb)
 		if err != nil {
 			uploadSpan.RecordError(err)
 			uploadSpan.SetStatus(codes.Error, err.Error())
@@ -588,7 +659,7 @@ func Deploy(ctx *Context, opts struct {
 		deployCtx, cancelDeploy := context.WithCancel(buildCtx)
 		defer cancelDeploy()
 
-		model := initialModel(updateCh, buildCh, uploadProgressCh)
+		model := initialModel(updateCh, buildCh, uploadProgressCh, cachedFiles, totalFiles)
 		p := tea.NewProgram(model)
 
 		var finalModel tea.Model
@@ -620,7 +691,7 @@ func Deploy(ctx *Context, opts struct {
 
 		cb = createBuildStatusCallback(deployCtx, updateCh, buildCh, &buildErrors, &buildLogs, progressHandler)
 
-		results, err = bc.BuildFromTar(deployCtx, name, stream.ServeReader(deployCtx, r), cb, envVars)
+		results, err = buildCall(deployCtx, r, cb)
 
 		// Ensure the progress UI is shut down before printing
 		p.Quit()
@@ -878,8 +949,14 @@ func buildStepsSummary(count int) string {
 	return fmt.Sprintf("%d steps completed", count)
 }
 
+// deployAccessInfo provides access to build result access info for display purposes.
+type deployAccessInfo interface {
+	HasAccessInfo() bool
+	AccessInfo() *build_v1alpha.AccessInfo
+}
+
 // displayAccessInfo shows how to access the deployed app using server-provided access info
-func displayAccessInfo(ctx *Context, appName string, results *build_v1alpha.BuilderClientBuildFromTarResults) {
+func displayAccessInfo(ctx *Context, appName string, results deployAccessInfo) {
 	// Check if we have access info from the server
 	if !results.HasAccessInfo() {
 		ctx.Log.Debug("No access info returned from server")

cli/commands/deploy_ui.go

@@ -113,6 +113,10 @@ type deployInfo struct {
 	uploadDuration   time.Duration
 	finalUploadSpeed float64
 
+	// Source cache info
+	cachedFiles int32
+	totalFiles  int
+
 	// Timeout and interrupt handling
 	lastActivity    time.Time
 	buildkitTimeout time.Duration
@@ -123,7 +127,7 @@ type deployInfo struct {
 	bp           tea.Model
 }
 
-func initialModel(update chan string, buildCh chan buildProgress, uploadProgress chan upload.Progress) *deployInfo {
+func initialModel(update chan string, buildCh chan buildProgress, uploadProgress chan upload.Progress, cachedFiles int32, totalFiles int) *deployInfo {
 	s := spinner.New()
 	s.Spinner = Meter
 	s.Style = lipgloss.NewStyle()
@@ -149,13 +153,28 @@ func initialModel(update chan string, buildCh chan buildProgress, uploadProgress
 		uploadSpeed:     "calculating...",
 		phaseStart:      time.Now(),
 		currentPhase:    "upload",
+		cachedFiles:     cachedFiles,
+		totalFiles:      totalFiles,
 		lastActivity:    time.Now(),
 		buildkitTimeout: 60 * time.Second, // 60 second timeout for buildkit to start
 		buildkitStarted: false,
 		bp:              progressui.TeaModel(),
 	}
 }
 
+func (m *deployInfo) uploadDetails() string {
+	var parts []string
+	if m.cachedFiles > 0 {
+		parts = append(parts, fmt.Sprintf("reused %d/%d files", m.cachedFiles, m.totalFiles))
+	}
+	if m.uploadBytes > 0 {
+		parts = append(parts, fmt.Sprintf("%s at %s",
+			upload.FormatBytes(m.uploadBytes),
+			upload.FormatSpeed(m.finalUploadSpeed)))
+	}
+	return strings.Join(parts, ", ")
+}
+
 func (m *deployInfo) Init() tea.Cmd {
 	cmds := []tea.Cmd{
 		m.bp.Init(),
@@ -244,20 +263,10 @@ func (m *deployInfo) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 				duration := time.Since(m.phaseStart)
 				m.uploadDuration = duration
 
-				// Create upload summary
-				var details string
-				if m.uploadBytes > 0 {
-					details = fmt.Sprintf("%s uploaded at %s",
-						upload.FormatBytes(m.uploadBytes),
-						upload.FormatSpeed(m.finalUploadSpeed))
-				} else if m.finalUploadSpeed > 0 {
-					details = fmt.Sprintf("Uploaded at %s", upload.FormatSpeed(m.finalUploadSpeed))
-				}
-
 				m.completedPhases = append(m.completedPhases, phaseSummary{
 					name:     "Upload artifacts",
 					duration: duration,
-					details:  details,
+					details:  m.uploadDetails(),
 				})
 
 				// Start tracking buildkit phase
@@ -305,17 +314,10 @@ func (m *deployInfo) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 			}
 
 			if !hasUploadPhase {
-				var details string
-				if m.uploadBytes > 0 {
-					details = fmt.Sprintf("%s uploaded at %s",
-						upload.FormatBytes(m.uploadBytes),
-						upload.FormatSpeed(m.finalUploadSpeed))
-				}
-
 				m.completedPhases = append(m.completedPhases, phaseSummary{
 					name:     "Upload artifacts",
 					duration: duration,
-					details:  details,
+					details:  m.uploadDetails(),
 				})
 			}
 

components/coordinate/coordinate.go

@@ -956,7 +956,7 @@ func (c *Coordinator) Start(ctx context.Context) error {
 	// Create app client for the builder
 	appClient := appclient.NewClient(c.Log, loopback)
 
-	bs := build.NewBuilder(c.Log, eac, appClient, addonsClient, c.Resolver, c.TempDir, c.LogWriter, c.CloudAuth.DNSHostname, c.BuildKit)
+	bs := build.NewBuilder(c.Log, eac, appClient, addonsClient, c.Resolver, c.TempDir, c.LogWriter, c.CloudAuth.DNSHostname, c.BuildKit, c.DataPath)
 	server.ExposeValue("dev.miren.runtime/build", build_v1alpha.AdaptBuilder(bs))
 
 	ls := logs.NewServer(c.Log, ec, c.Logs)