Refactor cache and update provenance to v0.2 (#198)

* Refactor cache * Update sigs.k8s.io/bom to v0.6.0 * Sync go work * Improve S3 * Always rely on the cache * Ensure build on osx * Improve format * Improve gitinfo * Fix build * Add tests
gitpod-io · Feb 25, 2025 · 292cb12 · 292cb12
1 parent cf69008
commit 292cb12
Showing 24 changed files with 1,734 additions and 1,048 deletions.
diff --git a/cmd/build.go b/cmd/build.go
@@ -12,6 +12,9 @@ import (
 	"time"
 
 	"github.com/gitpod-io/leeway/pkg/leeway"
+	"github.com/gitpod-io/leeway/pkg/leeway/cache"
+	"github.com/gitpod-io/leeway/pkg/leeway/cache/local"
+	"github.com/gitpod-io/leeway/pkg/leeway/cache/remote"
 	"github.com/gookit/color"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -84,7 +87,7 @@ var buildCmd = &cobra.Command{
 	},
 }
 
-func serveBuildResult(ctx context.Context, addr string, localCache *leeway.FilesystemCache, pkg *leeway.Package) {
+func serveBuildResult(ctx context.Context, addr string, localCache cache.LocalCache, pkg *leeway.Package) {
 	br, exists := localCache.Location(pkg)
 	if !exists {
 		log.Fatal("build result is not in local cache despite just being built. Something's wrong with the cache.")
@@ -121,7 +124,7 @@ func serveBuildResult(ctx context.Context, addr string, localCache *leeway.Files
 	}
 }
 
-func saveBuildResult(ctx context.Context, loc string, localCache *leeway.FilesystemCache, pkg *leeway.Package) {
+func saveBuildResult(ctx context.Context, loc string, localCache cache.LocalCache, pkg *leeway.Package) {
 	br, exists := localCache.Location(pkg)
 	if !exists {
 		log.Fatal("build result is not in local cache despite just being built. Something's wrong with the cache.")
@@ -178,20 +181,21 @@ func addBuildFlags(cmd *cobra.Command) {
 	cmd.Flags().Bool("report-github", os.Getenv("GITHUB_OUTPUT") != "", "Report package build success/failure to GitHub Actions using the GITHUB_OUTPUT environment variable")
 }
 
-func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, *leeway.FilesystemCache) {
+func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache) {
 	cm, _ := cmd.Flags().GetString("cache")
 	log.WithField("cacheMode", cm).Debug("configuring caches")
 	cacheLevel := leeway.CacheLevel(cm)
 
-	remoteCache := getRemoteCache()
+	var remoteCache cache.RemoteCache
 	switch cacheLevel {
 	case leeway.CacheNone, leeway.CacheLocal:
-		remoteCache = leeway.NoRemoteCache{}
+		remoteCache = remote.NewNoRemoteCache()
 	case leeway.CacheRemotePull:
-		remoteCache = &pullOnlyRemoteCache{C: remoteCache}
+		remoteCache = &pullOnlyRemoteCache{C: remote.NewNoRemoteCache()}
 	case leeway.CacheRemotePush:
-		remoteCache = &pushOnlyRemoteCache{C: remoteCache}
+		remoteCache = &pushOnlyRemoteCache{C: remote.NewNoRemoteCache()}
 	case leeway.CacheRemote:
+		remoteCache = remote.NewNoRemoteCache()
 	default:
 		log.Fatalf("invalid cache level: %s", cacheLevel)
 	}
@@ -212,7 +216,7 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, *leeway.FilesystemC
 		}
 	}
 	log.WithField("location", localCacheLoc).Debug("set up local cache")
-	localCache, err := leeway.NewFilesystemCache(localCacheLoc)
+	localCache, err := local.NewFilesystemCache(localCacheLoc)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -310,33 +314,33 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, *leeway.FilesystemC
 }
 
 type pushOnlyRemoteCache struct {
-	C leeway.RemoteCache
+	C cache.RemoteCache
 }
 
-func (c *pushOnlyRemoteCache) ExistingPackages(pkgs []*leeway.Package) (map[*leeway.Package]struct{}, error) {
-	return c.C.ExistingPackages(pkgs)
+func (c *pushOnlyRemoteCache) ExistingPackages(ctx context.Context, pkgs []cache.Package) (map[cache.Package]struct{}, error) {
+	return c.C.ExistingPackages(ctx, pkgs)
 }
 
-func (c *pushOnlyRemoteCache) Download(dst leeway.Cache, pkgs []*leeway.Package) error {
+func (c *pushOnlyRemoteCache) Download(ctx context.Context, dst cache.LocalCache, pkgs []cache.Package) error {
 	return nil
 }
 
-func (c *pushOnlyRemoteCache) Upload(src leeway.Cache, pkgs []*leeway.Package) error {
-	return c.C.Upload(src, pkgs)
+func (c *pushOnlyRemoteCache) Upload(ctx context.Context, src cache.LocalCache, pkgs []cache.Package) error {
+	return c.C.Upload(ctx, src, pkgs)
 }
 
 type pullOnlyRemoteCache struct {
-	C leeway.RemoteCache
+	C cache.RemoteCache
 }
 
-func (c *pullOnlyRemoteCache) ExistingPackages(pkgs []*leeway.Package) (map[*leeway.Package]struct{}, error) {
-	return c.C.ExistingPackages(pkgs)
+func (c *pullOnlyRemoteCache) ExistingPackages(ctx context.Context, pkgs []cache.Package) (map[cache.Package]struct{}, error) {
+	return c.C.ExistingPackages(ctx, pkgs)
 }
 
-func (c *pullOnlyRemoteCache) Download(dst leeway.Cache, pkgs []*leeway.Package) error {
-	return c.C.Download(dst, pkgs)
+func (c *pullOnlyRemoteCache) Download(ctx context.Context, dst cache.LocalCache, pkgs []cache.Package) error {
+	return c.C.Download(ctx, dst, pkgs)
 }
 
-func (c *pullOnlyRemoteCache) Upload(src leeway.Cache, pkgs []*leeway.Package) error {
+func (c *pullOnlyRemoteCache) Upload(ctx context.Context, src cache.LocalCache, pkgs []cache.Package) error {
 	return nil
 }
diff --git a/cmd/provenance-assert.go b/cmd/provenance-assert.go
@@ -73,7 +73,7 @@ var provenanceAssertCmd = &cobra.Command{
 				return nil
 			}
 
-			failures = append(assertions.AssertEnvelope(env), failures...)
+			failures = append(assertions.AssertBundle(env), failures...)
 
 			raw, err := base64.StdEncoding.DecodeString(env.Payload)
 			if err != nil {

diff --git a/cmd/root.go b/cmd/root.go
@@ -173,33 +173,6 @@ func getBuildArgs() (leeway.Arguments, error) {
 	return res, nil
 }
 
-func getRemoteCache() leeway.RemoteCache {
-	remoteCacheBucket := os.Getenv(EnvvarRemoteCacheBucket)
-	remoteStorage := os.Getenv(EnvvarRemoteCacheStorage)
-	if remoteCacheBucket != "" {
-		switch remoteStorage {
-		case "GCP":
-			return leeway.GSUtilRemoteCache{
-				BucketName: remoteCacheBucket,
-			}
-		case "AWS":
-			rc, err := leeway.NewS3RemoteCache(remoteCacheBucket, nil)
-			if err != nil {
-				log.Fatalf("cannot access remote S3 cache: %v", err)
-			}
-
-			return rc
-		default:
-			return leeway.GSUtilRemoteCache{
-				BucketName: remoteCacheBucket,
-			}
-		}
-
-	}
-
-	return leeway.NoRemoteCache{}
-}
-
 func addExperimentalCommand(parent, child *cobra.Command) {
 	if os.Getenv("LEEWAY_EXPERIMENTAL") != "true" {
 		return

diff --git a/go.mod b/go.mod
@@ -3,11 +3,10 @@ module github.com/gitpod-io/leeway
 go 1.23
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.32.3
-	github.com/aws/aws-sdk-go-v2/config v1.28.1
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2
-	github.com/aws/aws-sdk-go-v2/service/sts v1.32.3
+	github.com/aws/aws-sdk-go-v2 v1.36.1
+	github.com/aws/aws-sdk-go-v2/config v1.29.6
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.59
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.75.4
 	github.com/creack/pty v1.1.23
 	github.com/disiqueira/gotree v1.0.0
 	github.com/dop251/goja v0.0.0-20241024094426-79f3a7efcdbd
@@ -16,7 +15,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/gookit/color v1.5.4
 	github.com/imdario/mergo v0.3.13
-	github.com/in-toto/in-toto-golang v0.3.3
+	github.com/in-toto/in-toto-golang v0.9.0
 	github.com/karrick/godirwalk v1.17.0
 	github.com/minio/highwayhash v1.0.2
 	github.com/opencontainers/runc v1.1.10
@@ -26,49 +25,50 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.9.0
-	golang.org/x/mod v0.21.0
-	golang.org/x/sync v0.8.0
+	golang.org/x/mod v0.23.0
+	golang.org/x/sync v0.11.0
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da
 	gopkg.in/yaml.v3 v3.0.1
-	sigs.k8s.io/bom v0.1.0
+	sigs.k8s.io/bom v0.6.0
 )
 
 require (
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.42 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 // indirect
-	github.com/aws/smithy-go v1.22.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.59 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect
+	github.com/aws/smithy-go v1.22.2 // indirect
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
-	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
-	github.com/cyphar/filepath-securejoin v0.3.4 // indirect
+	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dlclark/regexp2 v1.11.4 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
-	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/godbus/dbus/v5 v5.0.6 // indirect
 	github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/kr/pretty v0.3.1 // indirect
-	github.com/moby/sys/mountinfo v0.7.1 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
+	github.com/moby/sys/mountinfo v0.5.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/rogpeppe/go-internal v1.11.0 // indirect
-	github.com/seccomp/libseccomp-golang v0.10.0 // indirect
+	github.com/rogpeppe/go-internal v1.13.1 // indirect
+	github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect
+	github.com/secure-systems-lab/go-securesystemslib v0.6.0 // indirect
 	github.com/segmentio/backo-go v1.0.0 // indirect
-	github.com/shibumi/go-pathspec v1.2.0 // indirect
+	github.com/shibumi/go-pathspec v1.3.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
-	golang.org/x/net v0.24.0 // indirect
-	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/crypto v0.18.0 // indirect
+	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	sigs.k8s.io/release-utils v0.3.0 // indirect
+	sigs.k8s.io/release-utils v0.7.7 // indirect
 )