diff --git a/.gitattributes b/.gitattributes index e4fb95146..babd135fd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,3 +5,4 @@ **/zz_deepcopy_generated.go linguist-generated=true cmd/crane/doc/crane*.md linguist-generated=true go.sum linguist-generated=true +**/testdata/** ignore-lint=true diff --git a/cmd/crane/cmd/gc.go b/cmd/crane/cmd/gc.go new file mode 100644 index 000000000..a80f14843 --- /dev/null +++ b/cmd/crane/cmd/gc.go @@ -0,0 +1,66 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "os" + + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/spf13/cobra" +) + +func NewCmdLayout() *cobra.Command { + cmd := &cobra.Command{ + Use: "layout", + } + cmd.AddCommand(newCmdGc()) + return cmd +} + +// NewCmdGc creates a new cobra.Command for the pull subcommand. +func newCmdGc() *cobra.Command { + cmd := &cobra.Command{ + Use: "gc OCI-LAYOUT", + Short: "Garbage collect unreferenced blobs in a local oci-layout", + Args: cobra.ExactArgs(1), + Hidden: true, // TODO: promote to public once theres some milage + RunE: func(_ *cobra.Command, args []string) error { + path := args[0] + + p, err := layout.FromPath(path) + + if err != nil { + return err + } + + blobs, err := p.GarbageCollect() + if err != nil { + return err + } + + for _, blob := range blobs { + if err := p.RemoveBlob(blob); err != nil { + return err + } + fmt.Fprintf(os.Stderr, "garbage collecting: %s\n", blob.String()) + } + + return nil + }, + } + + return cmd +} diff --git a/cmd/crane/cmd/root.go b/cmd/crane/cmd/root.go index 3faa85f5f..8fc8ccefa 100644 --- a/cmd/crane/cmd/root.go +++ b/cmd/crane/cmd/root.go @@ -129,7 +129,8 @@ func New(use, short string, options []crane.Option) *cobra.Command { NewCmdTag(&options), NewCmdValidate(&options), NewCmdVersion(), - newCmdRegistry(), + NewCmdRegistry(), + NewCmdLayout(), ) root.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Enable debug logs") diff --git a/cmd/crane/cmd/serve.go b/cmd/crane/cmd/serve.go index 61d031b39..5256c3714 100644 --- a/cmd/crane/cmd/serve.go +++ b/cmd/crane/cmd/serve.go @@ -28,7 +28,7 @@ import ( "github.com/google/go-containerregistry/pkg/registry" ) -func newCmdRegistry() *cobra.Command { +func NewCmdRegistry() *cobra.Command { cmd := &cobra.Command{ Use: "registry", } diff --git a/pkg/v1/layout/gc.go b/pkg/v1/layout/gc.go new file mode 100644 index 000000000..5fdb2c05d --- /dev/null +++ b/pkg/v1/layout/gc.go @@ -0,0 +1,137 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This is an EXPERIMENTAL package, and may change in arbitrary ways without notice. +package layout + +import ( + "fmt" + "io/fs" + "path/filepath" + "strings" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// GarbageCollect removes unreferenced blobs from the oci-layout +// +// This is an experimental api, and not subject to any stability guarantees +// We may abandon it at any time, without prior notice. +// Deprecated: Use it at your own risk! +func (l Path) GarbageCollect() ([]v1.Hash, error) { + idx, err := l.ImageIndex() + if err != nil { + return nil, err + } + blobsToKeep := map[string]bool{} + if err := l.garbageCollectImageIndex(idx, blobsToKeep); err != nil { + return nil, err + } + blobsDir := l.path("blobs") + removedBlobs := []v1.Hash{} + + err = filepath.WalkDir(blobsDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if d.IsDir() { + return nil + } + + rel, err := filepath.Rel(blobsDir, path) + if err != nil { + return err + } + hashString := strings.Replace(rel, "/", ":", 1) + if present := blobsToKeep[hashString]; !present { + h, err := v1.NewHash(hashString) + if err != nil { + return err + } + removedBlobs = append(removedBlobs, h) + } + return nil + }) + + if err != nil { + return nil, err + } + + return removedBlobs, nil +} + +func (l Path) garbageCollectImageIndex(index v1.ImageIndex, blobsToKeep map[string]bool) error { + idxm, err := index.IndexManifest() + if err != nil { + return err + } + + h, err := index.Digest() + if err != nil { + return err + } + + blobsToKeep[h.String()] = true + + for _, descriptor := range idxm.Manifests { + if descriptor.MediaType.IsImage() { + img, err := index.Image(descriptor.Digest) + if err != nil { + return err + } + if err := l.garbageCollectImage(img, blobsToKeep); err != nil { + return err + } + } else if descriptor.MediaType.IsIndex() { + idx, err := index.ImageIndex(descriptor.Digest) + if err != nil { + return err + } + if err := l.garbageCollectImageIndex(idx, blobsToKeep); err != nil { + return err + } + } else { + return fmt.Errorf("gc: unknown media type: %s", descriptor.MediaType) + } + } + return nil +} + +func (l Path) garbageCollectImage(image v1.Image, blobsToKeep map[string]bool) error { + h, err := image.Digest() + if err != nil { + return err + } + blobsToKeep[h.String()] = true + + h, err = image.ConfigName() + if err != nil { + return err + } + blobsToKeep[h.String()] = true + + ls, err := image.Layers() + if err != nil { + return err + } + for _, l := range ls { + h, err := l.Digest() + if err != nil { + return err + } + blobsToKeep[h.String()] = true + } + return nil +} diff --git a/pkg/v1/layout/gc_test.go b/pkg/v1/layout/gc_test.go new file mode 100644 index 000000000..c80a2dc42 --- /dev/null +++ b/pkg/v1/layout/gc_test.go @@ -0,0 +1,96 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "path/filepath" + "testing" +) + +var ( + gcIndexPath = filepath.Join("testdata", "test_gc_index") + gcIndexBlobHash = "sha256:492b89b9dd3cda4596f94916d17f6901455fb8bd7f4c5a2a90df8d39c90f48a0" + gcUnknownMediaTypePath = filepath.Join("testdata", "test_gc_image_unknown_mediatype") + gcUnknownMediaTypeErr = "gc: unknown media type: application/vnd.oci.descriptor.v1+json" + gcTestOneImagePath = filepath.Join("testdata", "test_index_one_image") + gcTestIndexMediaTypePath = filepath.Join("testdata", "test_index_media_type") +) + +func TestGcIndex(t *testing.T) { + lp, err := FromPath(gcIndexPath) + if err != nil { + t.Fatalf("FromPath() = %v", err) + } + + removed, err := lp.GarbageCollect() + if err != nil { + t.Fatalf("GarbageCollect() = %v", err) + } + + if len(removed) != 1 { + t.Fatalf("expected to have only one gc-able blob") + } + if removed[0].String() != gcIndexBlobHash { + t.Fatalf("wrong blob is gc-ed: expected '%s', got '%s'", gcIndexBlobHash, removed[0].String()) + } +} + +func TestGcOneImage(t *testing.T) { + lp, err := FromPath(gcTestOneImagePath) + if err != nil { + t.Fatalf("FromPath() = %v", err) + } + + removed, err := lp.GarbageCollect() + if err != nil { + t.Fatalf("GarbageCollect() = %v", err) + } + + if len(removed) != 0 { + t.Fatalf("expected to have to gc-able blobs") + } +} + +func TestGcIndexMediaType(t *testing.T) { + lp, err := FromPath(gcTestIndexMediaTypePath) + if err != nil { + t.Fatalf("FromPath() = %v", err) + } + + removed, err := lp.GarbageCollect() + if err != nil { + t.Fatalf("GarbageCollect() = %v", err) + } + + if len(removed) != 0 { + t.Fatalf("expected to have to gc-able blobs") + } +} + +func TestGcUnknownMediaType(t *testing.T) { + lp, err := FromPath(gcUnknownMediaTypePath) + if err != nil { + t.Fatalf("FromPath() = %v", err) + } + + _, err = lp.GarbageCollect() + if err == nil { + t.Fatalf("expected GarbageCollect to return err but did not") + } + + if err.Error() != gcUnknownMediaTypeErr { + t.Fatalf("expected error '%s', got '%s'", gcUnknownMediaTypeErr, err.Error()) + } +} diff --git a/pkg/v1/layout/testdata/test_gc_image_unknown_mediatype/index.json b/pkg/v1/layout/testdata/test_gc_image_unknown_mediatype/index.json new file mode 100644 index 000000000..7a8c41040 --- /dev/null +++ b/pkg/v1/layout/testdata/test_gc_image_unknown_mediatype/index.json @@ -0,0 +1,10 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.descriptor.v1+json", + "size": 423, + "digest": "sha256:32589985702551b6c56033bb3334432a0a513bf9d6aceda0f67c42b003850720" + } + ] +} diff --git a/pkg/v1/layout/testdata/test_gc_image_unknown_mediatype/oci-layout b/pkg/v1/layout/testdata/test_gc_image_unknown_mediatype/oci-layout new file mode 100644 index 000000000..10ff2f3ce --- /dev/null +++ b/pkg/v1/layout/testdata/test_gc_image_unknown_mediatype/oci-layout @@ -0,0 +1,3 @@ +{ + "imageLayoutVersion": "1.0.0" +} diff --git a/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5 b/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5 new file mode 100644 index 000000000..1597d0721 --- /dev/null +++ b/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5 @@ -0,0 +1,13 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 423, + "digest": "sha256:eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650", + "annotations": { + "org.opencontainers.image.ref.name": "1" + } + } + ] +} diff --git a/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/2b29a2b8dea3af91ea7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb b/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/2b29a2b8dea3af91ea7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb new file mode 100644 index 000000000..e6587e23e --- /dev/null +++ b/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/2b29a2b8dea3af91ea7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb @@ -0,0 +1,13 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 423, + "digest": "sha256:eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650", + "annotations": { + "org.opencontainers.image.ref.name": "4" + } + } + ] +} diff --git a/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/492b89b9dd3cda4596f94916d17f6901455fb8bd7f4c5a2a90df8d39c90f48a0 b/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/492b89b9dd3cda4596f94916d17f6901455fb8bd7f4c5a2a90df8d39c90f48a0 new file mode 100644 index 000000000..1e4eb2219 Binary files /dev/null and b/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/492b89b9dd3cda4596f94916d17f6901455fb8bd7f4c5a2a90df8d39c90f48a0 differ diff --git a/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e b/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e new file mode 100644 index 000000000..4228c8902 --- /dev/null +++ b/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e @@ -0,0 +1 @@ +{"architecture": "amd64", "author": "Bazel", "config": {}, "created": "1970-01-01T00:00:00Z", "history": [{"author": "Bazel", "created": "1970-01-01T00:00:00Z", "created_by": "bazel build ..."}], "os": "linux", "rootfs": {"diff_ids": ["sha256:8897395fd26dc44ad0e2a834335b33198cb41ac4d98dfddf58eced3853fa7b17"], "type": "layers"}} diff --git a/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b b/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b new file mode 100644 index 000000000..05c63217b Binary files /dev/null and b/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b differ diff --git a/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650 b/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650 new file mode 100644 index 000000000..21dc412c3 --- /dev/null +++ b/pkg/v1/layout/testdata/test_gc_index/blobs/sha256/eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650 @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","size":330,"digest":"sha256:6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e"},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","size":167,"digest":"sha256:dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b"}]} \ No newline at end of file diff --git a/pkg/v1/layout/testdata/test_gc_index/index.json b/pkg/v1/layout/testdata/test_gc_index/index.json new file mode 100644 index 000000000..9b6576c02 --- /dev/null +++ b/pkg/v1/layout/testdata/test_gc_index/index.json @@ -0,0 +1,29 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 423, + "digest": "sha256:eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650", + "annotations": { + "org.opencontainers.image.ref.name": "1" + } + }, + { + "mediaType": "application/vnd.oci.image.index.v1+json", + "size": 314, + "digest": "sha256:05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5", + "annotations": { + "org.opencontainers.image.ref.name": "3" + } + }, + { + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "size": 314, + "digest": "sha256:2b29a2b8dea3af91ea7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb", + "annotations": { + "org.opencontainers.image.ref.name": "4" + } + } + ] +} diff --git a/pkg/v1/layout/testdata/test_gc_index/oci-layout b/pkg/v1/layout/testdata/test_gc_index/oci-layout new file mode 100644 index 000000000..10ff2f3ce --- /dev/null +++ b/pkg/v1/layout/testdata/test_gc_index/oci-layout @@ -0,0 +1,3 @@ +{ + "imageLayoutVersion": "1.0.0" +}