Skip to content

Commit 0832a68

Browse files
leodidoona-agent
andcommitted
feat(docker): use OCI layout for deterministic image caching
Replace 'docker save' with 'docker buildx build --output type=oci' to achieve fully deterministic Docker image caching. Problem: - docker save creates symlinks for duplicate layers with non-deterministic timestamps (moby/moby#42766) - This breaks SLSA L3 compliance (provenance digest doesn't match artifact) - Cache verification fails due to different checksums on each build Solution: - Use OCI layout format (content-addressed, no symlinks) - Export directly from buildx (no intermediate docker save step) - Maintains backward compatibility (docker load supports OCI format) Changes: - Modify build command to use 'docker buildx build --output type=oci' when exportToCache is enabled - Remove 'docker save' command (buildx creates image.tar directly) - Add integration test for determinism verification - Update README with OCI layout documentation Benefits: - ✅ Fully deterministic builds (same source → same checksum) - ✅ SLSA L3 compliance (provenance verification works) - ✅ Standard format (OCI spec, widely supported) - ✅ Backward compatible (docker load supports both formats) - ✅ Smaller artifacts (content-addressed, no duplicate layers) Testing: - Added TestDockerPackage_OCILayout_Determinism_Integration - Verifies identical checksums across builds with same source Co-authored-by: Ona <no-reply@ona.com>
1 parent 2667fac commit 0832a68

File tree

3 files changed

+199
-7
lines changed

3 files changed

+199
-7
lines changed

README.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,18 @@ FROM alpine:3.18
362362
ARG SOURCE_DATE_EPOCH
363363
COPY --from=builder /app /app
364364
```
365+
366+
**OCI Layout for deterministic caching:**
367+
368+
When `exportToCache` is enabled, Docker images are exported in OCI layout format instead of using `docker save`. This ensures fully deterministic cache artifacts:
369+
370+
- **Format**: OCI Image Layout (standard)
371+
- **Loading**: `docker load -i image.tar` (automatic, backward compatible)
372+
- **Benefit**: Same source code produces identical cache checksums
373+
- **SLSA L3**: Enables provenance verification with matching digests
374+
375+
The OCI layout format is content-addressed and eliminates the non-deterministic symlink timestamps that occur with `docker save`.
376+
365377
## Package Variants
366378
Leeway supports build-time variance through "package variants". Those variants are defined on the workspace level and can modify the list of sources, environment variables and config of packages.
367379
For example consider a `WORKSPACE.YAML` with this variants section:

pkg/leeway/build.go

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1984,7 +1984,19 @@ func (p *Package) buildDocker(buildctx *buildContext, wd, result string) (res *p
19841984
return nil, err
19851985
}
19861986

1987-
buildcmd := []string{"docker", "build", "--pull", "-t", version}
1987+
// Use buildx for OCI layout export when exporting to cache
1988+
var buildcmd []string
1989+
if *cfg.ExportToCache {
1990+
// Build with OCI layout export for deterministic caching
1991+
imageTarPath := filepath.Join(wd, "image.tar")
1992+
buildcmd = []string{"docker", "buildx", "build", "--pull"}
1993+
buildcmd = append(buildcmd, "--output", fmt.Sprintf("type=oci,dest=%s", imageTarPath))
1994+
buildcmd = append(buildcmd, "--tag", version)
1995+
} else {
1996+
// Normal build (load to daemon for pushing)
1997+
buildcmd = []string{"docker", "build", "--pull", "-t", version}
1998+
}
1999+
19882000
for arg, val := range cfg.BuildArgs {
19892001
buildcmd = append(buildcmd, "--build-arg", fmt.Sprintf("%s=%s", arg, val))
19902002
}
@@ -2163,13 +2175,10 @@ func (p *Package) buildDocker(buildctx *buildContext, wd, result string) (res *p
21632175
res.Subjects = createDockerSubjectsFunction(version, cfg)
21642176
} else if len(cfg.Image) > 0 && *cfg.ExportToCache {
21652177
// Export to cache for signing
2166-
log.WithField("package", p.FullName()).Debug("Exporting Docker image to cache")
2178+
log.WithField("package", p.FullName()).Debug("Exporting Docker image to cache (OCI layout)")
21672179

2168-
// Export the image to tar
2169-
imageTarPath := filepath.Join(wd, "image.tar")
2170-
pkgCommands = append(pkgCommands,
2171-
[]string{"docker", "save", version, "-o", imageTarPath},
2172-
)
2180+
// Note: image.tar is already created by buildx --output type=oci
2181+
// No docker save needed!
21732182

21742183
// Store image names for later use
21752184
for _, img := range cfg.Image {

pkg/leeway/build_integration_test.go

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package leeway
66
import (
77
"archive/tar"
88
"compress/gzip"
9+
"crypto/sha256"
910
"encoding/json"
1011
"errors"
1112
"fmt"
@@ -529,3 +530,173 @@ CMD ["cat", "/test-file.txt"]`
529530

530531
t.Log("✅ Round-trip test passed: image exported, cached, extracted, loaded, and executed successfully")
531532
}
533+
534+
func TestDockerPackage_OCILayout_Determinism_Integration(t *testing.T) {
535+
if testing.Short() {
536+
t.Skip("Skipping integration test in short mode")
537+
}
538+
539+
// Ensure Docker is available
540+
if err := exec.Command("docker", "version").Run(); err != nil {
541+
t.Skip("Docker not available, skipping integration test")
542+
}
543+
544+
// Ensure buildx is available
545+
if err := exec.Command("docker", "buildx", "version").Run(); err != nil {
546+
t.Skip("Docker buildx not available, skipping integration test")
547+
}
548+
549+
// Create test workspace
550+
tmpDir := t.TempDir()
551+
wsDir := filepath.Join(tmpDir, "workspace")
552+
if err := os.MkdirAll(wsDir, 0755); err != nil {
553+
t.Fatal(err)
554+
}
555+
556+
// Create WORKSPACE.yaml
557+
workspaceYAML := `defaultTarget: ":test-image"`
558+
if err := os.WriteFile(filepath.Join(wsDir, "WORKSPACE.yaml"), []byte(workspaceYAML), 0644); err != nil {
559+
t.Fatal(err)
560+
}
561+
562+
// Create Dockerfile with ARG SOURCE_DATE_EPOCH
563+
dockerfile := `FROM alpine:3.18
564+
ARG SOURCE_DATE_EPOCH
565+
RUN echo "Build time: $SOURCE_DATE_EPOCH" > /build-time.txt
566+
CMD ["cat", "/build-time.txt"]
567+
`
568+
if err := os.WriteFile(filepath.Join(wsDir, "Dockerfile"), []byte(dockerfile), 0644); err != nil {
569+
t.Fatal(err)
570+
}
571+
572+
// Create BUILD.yaml
573+
buildYAML := `packages:
574+
- name: test-image
575+
type: docker
576+
config:
577+
dockerfile: Dockerfile
578+
image:
579+
- localhost/leeway-determinism-test:latest
580+
exportToCache: true
581+
`
582+
if err := os.WriteFile(filepath.Join(wsDir, "BUILD.yaml"), []byte(buildYAML), 0644); err != nil {
583+
t.Fatal(err)
584+
}
585+
586+
// Initialize git repo (required for deterministic mtime)
587+
gitInit := exec.Command("git", "init")
588+
gitInit.Dir = wsDir
589+
if err := gitInit.Run(); err != nil {
590+
t.Fatal(err)
591+
}
592+
593+
gitAdd := exec.Command("git", "add", ".")
594+
gitAdd.Dir = wsDir
595+
if err := gitAdd.Run(); err != nil {
596+
t.Fatal(err)
597+
}
598+
599+
gitCommit := exec.Command("git", "commit", "-m", "initial")
600+
gitCommit.Dir = wsDir
601+
if err := gitCommit.Run(); err != nil {
602+
t.Fatal(err)
603+
}
604+
605+
// Build first time
606+
cacheDir1 := filepath.Join(tmpDir, "cache1")
607+
if err := os.MkdirAll(cacheDir1, 0755); err != nil {
608+
t.Fatal(err)
609+
}
610+
611+
buildCtx1 := &buildContext{
612+
LocalCache: &FilesystemCache{Location: cacheDir1},
613+
DockerExportToCache: true,
614+
DockerExportSet: true,
615+
Reporter: &ConsoleReporter{},
616+
}
617+
618+
ws1, err := Load(wsDir)
619+
if err != nil {
620+
t.Fatalf("Failed to load workspace: %v", err)
621+
}
622+
623+
pkg1, exists := ws1.Packages[":test-image"]
624+
if !exists {
625+
t.Fatal("Package :test-image not found")
626+
}
627+
628+
if _, err := pkg1.build(buildCtx1); err != nil {
629+
t.Fatalf("First build failed: %v", err)
630+
}
631+
632+
// Get checksum of first build
633+
cacheFiles1, err := filepath.Glob(filepath.Join(cacheDir1, "*.tar.gz"))
634+
if err != nil || len(cacheFiles1) == 0 {
635+
t.Fatal("No cache file found after first build")
636+
}
637+
checksum1, err := checksumFile(cacheFiles1[0])
638+
if err != nil {
639+
t.Fatalf("Failed to checksum first build: %v", err)
640+
}
641+
642+
// Build second time (clean cache)
643+
cacheDir2 := filepath.Join(tmpDir, "cache2")
644+
if err := os.MkdirAll(cacheDir2, 0755); err != nil {
645+
t.Fatal(err)
646+
}
647+
648+
buildCtx2 := &buildContext{
649+
LocalCache: &FilesystemCache{Location: cacheDir2},
650+
DockerExportToCache: true,
651+
DockerExportSet: true,
652+
Reporter: &ConsoleReporter{},
653+
}
654+
655+
ws2, err := Load(wsDir)
656+
if err != nil {
657+
t.Fatalf("Failed to load workspace: %v", err)
658+
}
659+
660+
pkg2, exists := ws2.Packages[":test-image"]
661+
if !exists {
662+
t.Fatal("Package :test-image not found")
663+
}
664+
665+
if _, err := pkg2.build(buildCtx2); err != nil {
666+
t.Fatalf("Second build failed: %v", err)
667+
}
668+
669+
// Get checksum of second build
670+
cacheFiles2, err := filepath.Glob(filepath.Join(cacheDir2, "*.tar.gz"))
671+
if err != nil || len(cacheFiles2) == 0 {
672+
t.Fatal("No cache file found after second build")
673+
}
674+
checksum2, err := checksumFile(cacheFiles2[0])
675+
if err != nil {
676+
t.Fatalf("Failed to checksum second build: %v", err)
677+
}
678+
679+
// Compare checksums
680+
if checksum1 != checksum2 {
681+
t.Errorf("Builds are not deterministic!\nBuild 1: %s\nBuild 2: %s", checksum1, checksum2)
682+
t.Log("This indicates the OCI layout export is not fully deterministic")
683+
} else {
684+
t.Logf("✅ Deterministic builds verified: %s", checksum1)
685+
}
686+
}
687+
688+
// checksumFile computes SHA256 checksum of a file
689+
func checksumFile(path string) (string, error) {
690+
f, err := os.Open(path)
691+
if err != nil {
692+
return "", err
693+
}
694+
defer f.Close()
695+
696+
h := sha256.New()
697+
if _, err := io.Copy(h, f); err != nil {
698+
return "", err
699+
}
700+
701+
return fmt.Sprintf("%x", h.Sum(nil)), nil
702+
}

0 commit comments

Comments
 (0)