fix: upload and download provenance bundles in S3 cache

pkg/leeway/cache/remote/s3.go

@@ -578,6 +578,9 @@ func (s *S3Cache) downloadWithSLSAVerification(ctx context.Context, p cache.Pack
 			continue
 		}
 
+		// Step 6: Download provenance bundle if it exists (best effort, non-blocking)
+		s.downloadProvenanceBundle(ctx, p.FullName(), artifactKey, localPath)
+
 		// Clean up temporary attestation file
 		s.cleanupTempFiles(tmpAttestationPath)
 
@@ -967,6 +970,10 @@ func (s *S3Cache) Upload(ctx context.Context, src cache.LocalCache, pkgs []cache
 			"package": p.FullName(),
 			"key":     key,
 		}).Debug("successfully uploaded package to remote cache")
+
+		// Upload provenance bundle if it exists (non-blocking)
+		s.uploadProvenanceBundle(ctx, p.FullName(), key, localPath)
+
 		return nil
 	})
 
@@ -1222,3 +1229,127 @@ func (s *S3Storage) ListObjects(ctx context.Context, prefix string) ([]string, e
 
 	return result, nil
 }
+
+// fileExists checks if a file exists and is not a directory
+func fileExists(filename string) bool {
+	info, err := os.Stat(filename)
+	if err != nil {
+		return false
+	}
+	return !info.IsDir()
+}
+
+// uploadProvenanceBundle uploads a provenance bundle to S3 with retry logic.
+// This is a non-blocking operation - failures are logged but don't fail the build.
+// Provenance bundles are stored alongside artifacts as <artifact>.provenance.jsonl
+// and are needed for dependency provenance collection during local builds.
+func (s *S3Cache) uploadProvenanceBundle(ctx context.Context, packageName, artifactKey, localPath string) {
+	provenancePath := localPath + ".provenance.jsonl"
+
+	// Check if provenance file exists
+	if !fileExists(provenancePath) {
+		log.WithFields(log.Fields{
+			"package": packageName,
+			"path":    provenancePath,
+		}).Debug("Provenance bundle not found locally, skipping upload")
+		return
+	}
+
+	provenanceKey := artifactKey + ".provenance.jsonl"
+
+	// Wait for rate limiter permission
+	if err := s.waitForRateLimit(ctx); err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"package": packageName,
+			"key":     provenanceKey,
+		}).Warn("Rate limiter error during provenance upload, skipping")
+		return
+	}
+
+	// Upload with timeout and retry logic (via storage layer)
+	uploadCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
+	defer cancel()
+
+	if err := s.storage.UploadObject(uploadCtx, provenanceKey, provenancePath); err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"package": packageName,
+			"key":     provenanceKey,
+			"path":    provenancePath,
+		}).Warn("Failed to upload provenance bundle to remote cache")
+		return
+	}
+
+	log.WithFields(log.Fields{
+		"package": packageName,
+		"key":     provenanceKey,
+	}).Debug("Successfully uploaded provenance bundle to remote cache")
+}
+
+// downloadProvenanceBundle downloads a provenance bundle from S3 with verification.
+// This is a best-effort operation - missing provenance is expected for older artifacts.
+// Returns true if provenance was successfully downloaded, false otherwise.
+func (s *S3Cache) downloadProvenanceBundle(ctx context.Context, packageName, artifactKey, localPath string) bool {
+	provenanceKey := artifactKey + ".provenance.jsonl"
+	provenancePath := localPath + ".provenance.jsonl"
+	tmpProvenancePath := provenancePath + ".tmp"
+
+	// Wait for rate limiter permission
+	if err := s.waitForRateLimit(ctx); err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"package": packageName,
+			"key":     provenanceKey,
+		}).Debug("Rate limiter error during provenance download, skipping")
+		return false
+	}
+
+	// Download with timeout
+	downloadCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	bytesDownloaded, err := s.storage.GetObject(downloadCtx, provenanceKey, tmpProvenancePath)
+	if err != nil {
+		// Provenance not found - this is expected for older artifacts
+		log.WithFields(log.Fields{
+			"package": packageName,
+			"key":     provenanceKey,
+		}).Debug("Provenance bundle not found in remote cache (expected for older artifacts)")
+		s.cleanupTempFiles(tmpProvenancePath)
+		return false
+	}
+
+	// Verify the downloaded file exists and has content
+	if !fileExists(tmpProvenancePath) {
+		log.WithFields(log.Fields{
+			"package": packageName,
+			"key":     provenanceKey,
+		}).Warn("Provenance bundle download reported success but file not found")
+		s.cleanupTempFiles(tmpProvenancePath)
+		return false
+	}
+
+	if bytesDownloaded == 0 {
+		log.WithFields(log.Fields{
+			"package": packageName,
+			"key":     provenanceKey,
+		}).Warn("Provenance bundle downloaded but file is empty")
+		s.cleanupTempFiles(tmpProvenancePath)
+		return false
+	}
+
+	// Atomically move to final location
+	if err := s.atomicMove(tmpProvenancePath, provenancePath); err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"package": packageName,
+			"key":     provenanceKey,
+		}).Warn("Failed to move provenance bundle to final location")
+		s.cleanupTempFiles(tmpProvenancePath)
+		return false
+	}
+
+	log.WithFields(log.Fields{
+		"package": packageName,
+		"key":     provenanceKey,
+		"bytes":   bytesDownloaded,
+	}).Debug("Successfully downloaded provenance bundle")
+	return true
+}