diff --git a/pkg/cmd/build/build.go b/pkg/cmd/build/build.go index 25da8b96..5c48fe1a 100644 --- a/pkg/cmd/build/build.go +++ b/pkg/cmd/build/build.go @@ -2,13 +2,14 @@ package build import ( "context" + "fmt" "kitops/pkg/artifact" "kitops/pkg/lib/constants" "kitops/pkg/lib/filesystem" + "kitops/pkg/lib/repo" "kitops/pkg/lib/storage" "kitops/pkg/output" "os" - "path/filepath" ) func RunBuild(ctx context.Context, options *buildOptions) error { @@ -63,21 +64,23 @@ func RunBuild(ctx context.Context, options *buildOptions) error { } model.Layers = append(model.Layers, *layer) - modelStorePath := options.storageHome - repo := "" tag := "" if options.modelRef != nil { - repo = filepath.Join(options.modelRef.Registry, options.modelRef.Repository) tag = options.modelRef.Reference } - store := storage.NewLocalStore(modelStorePath, repo) - manifestDesc, err := store.SaveModel(ctx, model, tag) + storageHome := constants.StoragePath(options.configHome) + localStore, err := repo.NewLocalStore(storageHome, options.modelRef) + if err != nil { + return fmt.Errorf("failed to open local storage: %w", err) + } + + manifestDesc, err := storage.SaveModel(ctx, localStore, model, tag) if err != nil { return err } for _, tag := range options.extraRefs { - if err := store.TagModel(ctx, *manifestDesc, tag); err != nil { + if err := localStore.Tag(ctx, *manifestDesc, tag); err != nil { return err } } diff --git a/pkg/cmd/build/cmd.go b/pkg/cmd/build/cmd.go index 50e391e0..be4b5450 100644 --- a/pkg/cmd/build/cmd.go +++ b/pkg/cmd/build/cmd.go @@ -10,7 +10,7 @@ import ( "kitops/pkg/lib/constants" "kitops/pkg/lib/filesystem" - "kitops/pkg/lib/storage" + "kitops/pkg/lib/repo" "kitops/pkg/output" "github.com/spf13/cobra" @@ -78,7 +78,7 @@ func (opts *buildOptions) complete(ctx context.Context, args []string) error { opts.storageHome = constants.StoragePath(opts.configHome) if opts.fullTagRef != "" { - modelRef, extraRefs, err := storage.ParseReference(opts.fullTagRef) + modelRef, extraRefs, err := repo.ParseReference(opts.fullTagRef) if err != nil { return fmt.Errorf("failed to parse reference %s: %w", opts.fullTagRef, err) } diff --git a/pkg/cmd/export/cmd.go b/pkg/cmd/export/cmd.go index 6f6592c7..8683324c 100644 --- a/pkg/cmd/export/cmd.go +++ b/pkg/cmd/export/cmd.go @@ -7,7 +7,6 @@ import ( "kitops/pkg/cmd/options" "kitops/pkg/lib/constants" "kitops/pkg/lib/repo" - "kitops/pkg/lib/storage" "kitops/pkg/output" "github.com/spf13/cobra" @@ -44,7 +43,7 @@ func (opts *exportOptions) complete(ctx context.Context, args []string) error { return fmt.Errorf("default config path not set on command context") } opts.configHome = configHome - modelRef, extraTags, err := storage.ParseReference(args[0]) + modelRef, extraTags, err := repo.ParseReference(args[0]) if err != nil { return fmt.Errorf("failed to parse reference %s: %w", args[0], err) } @@ -112,7 +111,7 @@ func runCommand(opts *exportOptions) func(*cobra.Command, []string) { func getStoreForRef(ctx context.Context, opts *exportOptions) (oras.Target, error) { storageHome := constants.StoragePath(opts.configHome) - localStore, err := oci.New(storage.LocalStorePath(storageHome, opts.modelRef)) + localStore, err := oci.New(repo.RepoPath(storageHome, opts.modelRef)) if err != nil { return nil, fmt.Errorf("failed to read local storage: %s\n", err) } diff --git a/pkg/cmd/list/cmd.go b/pkg/cmd/list/cmd.go index 9395c217..145ef7a3 100644 --- a/pkg/cmd/list/cmd.go +++ b/pkg/cmd/list/cmd.go @@ -5,7 +5,7 @@ import ( "fmt" "kitops/pkg/cmd/options" "kitops/pkg/lib/constants" - "kitops/pkg/lib/storage" + "kitops/pkg/lib/repo" "kitops/pkg/output" "os" "text/tabwriter" @@ -32,7 +32,7 @@ func (opts *listOptions) complete(ctx context.Context, args []string) error { } opts.configHome = configHome if len(args) > 0 { - remoteRef, extraTags, err := storage.ParseReference(args[0]) + remoteRef, extraTags, err := repo.ParseReference(args[0]) if err != nil { return fmt.Errorf("invalid reference: %w", err) } diff --git a/pkg/cmd/list/list.go b/pkg/cmd/list/list.go index f866272d..25808973 100644 --- a/pkg/cmd/list/list.go +++ b/pkg/cmd/list/list.go @@ -5,15 +5,11 @@ package list import ( "context" - "encoding/json" "fmt" - "io/fs" "kitops/pkg/artifact" "kitops/pkg/lib/constants" - "kitops/pkg/lib/storage" + "kitops/pkg/lib/repo" "math" - "os" - "path/filepath" "strings" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -26,15 +22,13 @@ const ( func listLocalKits(ctx context.Context, opts *listOptions) ([]string, error) { storageRoot := constants.StoragePath(opts.configHome) - storeDirs, err := findRepos(storageRoot) + stores, err := repo.GetAllLocalStores(storageRoot) if err != nil { return nil, err } var allInfoLines []string - for _, storeDir := range storeDirs { - store := storage.NewLocalStore(storageRoot, storeDir) - + for _, store := range stores { infolines, err := listKits(ctx, store) if err != nil { return nil, err @@ -44,56 +38,25 @@ func listLocalKits(ctx context.Context, opts *listOptions) ([]string, error) { return allInfoLines, nil } -func listKits(ctx context.Context, store storage.Store) ([]string, error) { - index, err := store.ParseIndexJson() +func listKits(ctx context.Context, store repo.LocalStorage) ([]string, error) { + index, err := store.GetIndex() if err != nil { return nil, err } var infolines []string for _, manifestDesc := range index.Manifests { - manifest, err := getManifest(ctx, store, manifestDesc) - if err != nil { - return nil, err - } - if manifest.Config.MediaType != constants.ModelConfigMediaType { - continue - } - manifestConf, err := readManifestConfig(ctx, store, manifest) + manifest, config, err := repo.GetManifestAndConfig(ctx, store, manifestDesc) if err != nil { return nil, err } - infoline := getManifestInfoLine(store.GetRepository(), manifestDesc, manifest, manifestConf) + infoline := getManifestInfoLine(store.GetRepo(), manifestDesc, manifest, config) infolines = append(infolines, infoline) } return infolines, nil } -func getManifest(ctx context.Context, store storage.Store, manifestDesc ocispec.Descriptor) (*ocispec.Manifest, error) { - manifestBytes, err := store.Fetch(ctx, manifestDesc) - if err != nil { - return nil, fmt.Errorf("failed to read manifest %s: %w", manifestDesc.Digest, err) - } - manifest := &ocispec.Manifest{} - if err := json.Unmarshal(manifestBytes, &manifest); err != nil { - return nil, fmt.Errorf("failed to parse manifest %s: %w", manifestDesc.Digest, err) - } - return manifest, nil -} - -func readManifestConfig(ctx context.Context, store storage.Store, manifest *ocispec.Manifest) (*artifact.KitFile, error) { - configBytes, err := store.Fetch(ctx, manifest.Config) - if err != nil { - return nil, fmt.Errorf("failed to read config: %w", err) - } - config := &artifact.KitFile{} - if err := json.Unmarshal(configBytes, config); err != nil { - return nil, fmt.Errorf("failed to parse config: %w", err) - } - return config, nil -} - func getManifestInfoLine(repo string, desc ocispec.Descriptor, manifest *ocispec.Manifest, config *artifact.KitFile) string { ref := desc.Annotations[ocispec.AnnotationRefName] if ref == "" { @@ -122,34 +85,6 @@ func getManifestInfoLine(repo string, desc ocispec.Descriptor, manifest *ocispec return info } -func findRepos(storePath string) ([]string, error) { - var indexPaths []string - err := filepath.WalkDir(storePath, func(file string, info fs.DirEntry, err error) error { - if err != nil { - return err - } - if info.Name() == "index.json" && !info.IsDir() { - dir := filepath.Dir(file) - relDir, err := filepath.Rel(storePath, dir) - if err != nil { - return err - } - if relDir == "." { - relDir = "" - } - indexPaths = append(indexPaths, relDir) - } - return nil - }) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - return nil, fmt.Errorf("failed to read local storage: %w", err) - } - return indexPaths, nil -} - func formatBytes(i int64) string { if i == 0 { return "0 B" diff --git a/pkg/cmd/list/list_test.go b/pkg/cmd/list/list_test.go index 01abe00e..2a5352ba 100644 --- a/pkg/cmd/list/list_test.go +++ b/pkg/cmd/list/list_test.go @@ -1,182 +1,12 @@ package list import ( - "context" "fmt" - "kitops/pkg/artifact" - "kitops/pkg/lib/constants" - internal "kitops/pkg/lib/testing" "testing" - "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" ) -func TestListKits(t *testing.T) { - tests := []struct { - testName string - repo string - manifests map[digest.Digest]ocispec.Manifest - configs map[digest.Digest]artifact.KitFile - index *ocispec.Index - expectedOutputRegexps []string - expectErrRegexp string - }{ - { - testName: "Cannot read index.json", - index: nil, - expectErrRegexp: "artifact not found", - }, - { - testName: "Cannot find manifest from index.json", - index: &ocispec.Index{ - Manifests: []ocispec.Descriptor{ - ManifestDesc("manifestA", "", true), - ManifestDesc("manifestNotFound", "", true), - }, - }, - manifests: map[digest.Digest]ocispec.Manifest{ - "manifestA": Manifest("configA", "layerA"), - "manifestB": Manifest("configB", "layerB"), - }, - configs: map[digest.Digest]artifact.KitFile{ - "configA": Config("maintainerA", "formatA"), - "configB": Config("maintainerB", "formatB"), - }, - expectErrRegexp: "failed to read manifest manifestNotFound.*", - }, - { - testName: "Cannot find config in manifest", - index: &ocispec.Index{ - Manifests: []ocispec.Descriptor{ - ManifestDesc("manifestA", "", true), - ManifestDesc("manifestB", "", true), - }, - }, - manifests: map[digest.Digest]ocispec.Manifest{ - "manifestA": Manifest("configA", "layerA"), - "manifestB": Manifest("configNotFound", "layerB"), - }, - configs: map[digest.Digest]artifact.KitFile{ - "configA": Config("maintainerA", "formatA"), - "configB": Config("maintainerB", "formatB"), - }, - expectErrRegexp: "failed to read config.*", - }, - { - testName: "Catches invalid size error", - index: &ocispec.Index{ - Manifests: []ocispec.Descriptor{ - ManifestDesc("manifestA", "", true), - ManifestDesc("manifestB", "", false), - }, - }, - manifests: map[digest.Digest]ocispec.Manifest{ - "manifestA": Manifest("configA", "layerA"), - "manifestB": Manifest("configB", "layerB1", "layerB2", "layerB3"), - }, - configs: map[digest.Digest]artifact.KitFile{ - "configA": Config("maintainerA", "formatA"), - "configB": Config("maintainerB", "formatB"), - }, - expectErrRegexp: "failed to read manifest manifestB: invalid size", - }, - { - testName: "Prints summary of for each manifest line (layers are 1024 bytes)", - index: &ocispec.Index{ - Manifests: []ocispec.Descriptor{ - ManifestDesc("manifestA", "", true), - ManifestDesc("manifestB", "", true), - }, - }, - manifests: map[digest.Digest]ocispec.Manifest{ - "manifestA": Manifest("configA", "layerA"), - "manifestB": Manifest("configB", "layerB1", "layerB2", "layerB3"), - }, - configs: map[digest.Digest]artifact.KitFile{ - "configA": Config("maintainerA", "formatA"), - "configB": Config("maintainerB", "formatB"), - }, - expectedOutputRegexps: []string{ - "\t\tmaintainerA\tformatA\t1.0 KiB\tmanifestA\t", - "\t\tmaintainerB\tformatB\t3.0 KiB\tmanifestB\t", - }, - }, - { - testName: "Prints summary of for each manifest line including repo and tag", - repo: "testregistry/testrepo", - index: &ocispec.Index{ - Manifests: []ocispec.Descriptor{ - ManifestDesc("manifestA", "tagA", true), - ManifestDesc("manifestB", "tagB", true), - }, - }, - manifests: map[digest.Digest]ocispec.Manifest{ - "manifestA": Manifest("configA", "layerA"), - "manifestB": Manifest("configB", "layerB1", "layerB2", "layerB3"), - }, - configs: map[digest.Digest]artifact.KitFile{ - "configA": Config("maintainerA", "formatA"), - "configB": Config("maintainerB", "formatB"), - }, - expectedOutputRegexps: []string{ - "testregistry/testrepo\ttagA\tmaintainerA\tformatA\t1.0 KiB\tmanifestA\t", - "testregistry/testrepo\ttagB\tmaintainerB\tformatB\t3.0 KiB\tmanifestB\t", - }, - }, - { - testName: "Prints summary of for each manifest line, stripping localhost/ if present", - repo: "localhost/testrepo", - index: &ocispec.Index{ - Manifests: []ocispec.Descriptor{ - ManifestDesc("manifestA", "tagA", true), - ManifestDesc("manifestB", "", true), - }, - }, - manifests: map[digest.Digest]ocispec.Manifest{ - "manifestA": Manifest("configA", "layerA"), - "manifestB": Manifest("configB", "layerB1", "layerB2", "layerB3"), - }, - configs: map[digest.Digest]artifact.KitFile{ - "configA": Config("maintainerA", "formatA"), - "configB": Config("maintainerB", "formatB"), - }, - expectedOutputRegexps: []string{ - "testrepo\ttagA\tmaintainerA\tformatA\t1.0 KiB\tmanifestA\t", - "testrepo\t\tmaintainerB\tformatB\t3.0 KiB\tmanifestB\t", - }, - }, - } - for _, tt := range tests { - t.Run(tt.testName, func(t *testing.T) { - testStore := &internal.TestStore{ - Manifests: tt.manifests, - Configs: tt.configs, - Index: tt.index, - Repo: tt.repo, - } - summaryLines, err := listKits(context.Background(), testStore) - if tt.expectErrRegexp != "" { - // Should be error - assert.Empty(t, summaryLines, "Should not output summary on error") - if !assert.Error(t, err, "Should return an error") { - return - } - assert.Regexp(t, tt.expectErrRegexp, err.Error()) - } else { - if !assert.NoError(t, err, "Should not return an error") { - return - } - for _, line := range tt.expectedOutputRegexps { - // Assert all lines in expected output are somewhere in the summary - assert.Contains(t, summaryLines, line) - } - } - }) - } -} - func TestFormatBytes(t *testing.T) { tests := []struct { input int64 @@ -204,46 +34,3 @@ func TestFormatBytes(t *testing.T) { }) } } - -func Manifest(configDigest string, layerDigests ...string) ocispec.Manifest { - manifest := ocispec.Manifest{ - Config: ocispec.Descriptor{ - MediaType: constants.ModelConfigMediaType, - Digest: digest.Digest(configDigest), - }, - } - for _, layerDigest := range layerDigests { - manifest.Layers = append(manifest.Layers, ocispec.Descriptor{ - MediaType: constants.ModelLayerMediaType, - Digest: digest.Digest(layerDigest), - Size: 1024, - }) - } - - return manifest -} - -func Config(maintainer, name string) artifact.KitFile { - config := artifact.KitFile{ - Kit: artifact.ModelKit{Authors: []string{maintainer}, Name: name}, - } - - return config -} - -func ManifestDesc(digestStr string, tag string, valid bool) ocispec.Descriptor { - size := internal.ValidSize - if !valid { - size = internal.InvalidSize - } - desc := ocispec.Descriptor{ - Digest: digest.Digest(digestStr), - MediaType: ocispec.MediaTypeImageManifest, - Size: size, - Annotations: map[string]string{}, - } - if tag != "" { - desc.Annotations[ocispec.AnnotationRefName] = tag - } - return desc -} diff --git a/pkg/cmd/pull/cmd.go b/pkg/cmd/pull/cmd.go index 05d344d9..42621719 100644 --- a/pkg/cmd/pull/cmd.go +++ b/pkg/cmd/pull/cmd.go @@ -6,7 +6,6 @@ import ( "kitops/pkg/cmd/options" "kitops/pkg/lib/constants" "kitops/pkg/lib/repo" - "kitops/pkg/lib/storage" "kitops/pkg/output" "github.com/spf13/cobra" @@ -32,7 +31,7 @@ func (opts *pullOptions) complete(ctx context.Context, args []string) error { } opts.configHome = configHome - modelRef, extraTags, err := storage.ParseReference(args[0]) + modelRef, extraTags, err := repo.ParseReference(args[0]) if err != nil { return fmt.Errorf("failed to parse reference %s: %w", modelRef, err) } @@ -78,7 +77,7 @@ func runCommand(opts *pullOptions) func(*cobra.Command, []string) { } storageHome := constants.StoragePath(opts.configHome) - localStorePath := storage.LocalStorePath(storageHome, opts.modelRef) + localStorePath := repo.RepoPath(storageHome, opts.modelRef) localStore, err := oci.New(localStorePath) if err != nil { output.Fatalln(err) diff --git a/pkg/cmd/push/cmd.go b/pkg/cmd/push/cmd.go index 48c27c19..2b3b8fed 100644 --- a/pkg/cmd/push/cmd.go +++ b/pkg/cmd/push/cmd.go @@ -6,7 +6,6 @@ import ( "kitops/pkg/cmd/options" "kitops/pkg/lib/constants" "kitops/pkg/lib/repo" - "kitops/pkg/lib/storage" "kitops/pkg/output" "github.com/spf13/cobra" @@ -32,7 +31,7 @@ func (opts *pushOptions) complete(ctx context.Context, args []string) error { } opts.configHome = configHome - modelRef, extraTags, err := storage.ParseReference(args[0]) + modelRef, extraTags, err := repo.ParseReference(args[0]) if err != nil { return fmt.Errorf("failed to parse reference %s: %w", modelRef, err) } @@ -79,7 +78,7 @@ func runCommand(opts *pushOptions) func(*cobra.Command, []string) { } storageHome := constants.StoragePath(opts.configHome) - localStorePath := storage.LocalStorePath(storageHome, opts.modelRef) + localStorePath := repo.RepoPath(storageHome, opts.modelRef) localStore, err := oci.New(localStorePath) if err != nil { output.Fatalln(err) diff --git a/pkg/lib/constants/consts.go b/pkg/lib/constants/consts.go index 31a81189..e9216bd1 100644 --- a/pkg/lib/constants/consts.go +++ b/pkg/lib/constants/consts.go @@ -28,3 +28,7 @@ func StoragePath(configBase string) string { func CredentialsPath(configBase string) string { return filepath.Join(configBase, CredentialsSubpath) } + +func IndexJsonPath(configBase string) string { + return filepath.Join(configBase, "index.json") +} diff --git a/pkg/lib/repo/local.go b/pkg/lib/repo/local.go index 195fe7ea..aeebc297 100644 --- a/pkg/lib/repo/local.go +++ b/pkg/lib/repo/local.go @@ -1,5 +1,123 @@ package repo -func NewLocalStore() { +import ( + "encoding/json" + "fmt" + "io/fs" + "kitops/pkg/lib/constants" + "os" + "path" + "path/filepath" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/oci" + "oras.land/oras-go/v2/registry" +) + +type LocalStorage interface { + GetRepo() string + GetIndex() (*ocispec.Index, error) + oras.Target +} + +type LocalStore struct { + storePath string + repo string + *oci.Store +} + +func GetAllLocalStores(storageRoot string) ([]LocalStorage, error) { + subDirs, err := findStoragePaths(storageRoot) + if err != nil { + return nil, err + } + var stores []LocalStorage + for _, subDir := range subDirs { + // convert to forward slashes for repo + repo := filepath.ToSlash(subDir) + storePath := filepath.Join(storageRoot, subDir) + ociStore, err := oci.New(storePath) + if err != nil { + return nil, err + } + localStore := &LocalStore{ + storePath: storePath, + repo: repo, + Store: ociStore, + } + stores = append(stores, localStore) + } + return stores, nil +} + +func NewLocalStore(storageRoot string, ref *registry.Reference) (LocalStorage, error) { + storePath := storageRoot + repo := "" + if ref != nil { + repo = path.Join(ref.Registry, ref.Repository) + storePath = filepath.Join(storePath, ref.Registry, ref.Repository) + } + store, err := oci.New(storePath) + if err != nil { + return nil, err + } + return &LocalStore{ + storePath: storePath, + repo: repo, + Store: store, + }, nil +} + +func (s *LocalStore) GetIndex() (*ocispec.Index, error) { + return parseIndexJson(s.storePath) +} + +func (s *LocalStore) GetRepo() string { + return s.repo +} + +func findStoragePaths(storageRoot string) ([]string, error) { + var indexPaths []string + err := filepath.WalkDir(storageRoot, func(file string, info fs.DirEntry, err error) error { + if err != nil { + return err + } + if info.Name() == "index.json" && !info.IsDir() { + dir := filepath.Dir(file) + relDir, err := filepath.Rel(storageRoot, dir) + if err != nil { + return err + } + if relDir == "." { + relDir = "" + } + indexPaths = append(indexPaths, relDir) + } + return nil + }) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("failed to read local storage: %w", err) + } + return indexPaths, nil +} + +func parseIndexJson(storageHome string) (*ocispec.Index, error) { + indexBytes, err := os.ReadFile(constants.IndexJsonPath(storageHome)) + if err != nil { + if os.IsNotExist(err) { + return &ocispec.Index{}, nil + } + return nil, fmt.Errorf("failed to read index: %w", err) + } + + index := &ocispec.Index{} + if err := json.Unmarshal(indexBytes, index); err != nil { + return nil, fmt.Errorf("failed to parse index: %w", err) + } + + return index, nil } diff --git a/pkg/lib/repo/repo.go b/pkg/lib/repo/repo.go index 051e32d2..73aba61c 100644 --- a/pkg/lib/repo/repo.go +++ b/pkg/lib/repo/repo.go @@ -6,11 +6,41 @@ import ( "fmt" "kitops/pkg/artifact" "kitops/pkg/lib/constants" + "path/filepath" + "regexp" + "strings" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/content" + "oras.land/oras-go/v2/registry" ) +var ( + validTagRegex = regexp.MustCompile(`^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$`) +) + +// ParseReference parses a reference string into a Reference struct. If the +// reference does not include a registry (e.g. myrepo:mytag), the placeholder +// 'localhost' is used. Additional tags can be specified in a comma-separated +// list (e.g. myrepo:tag1,tag2,tag3) +func ParseReference(refString string) (ref *registry.Reference, extraTags []string, err error) { + // References _must_ contain host; use localhost to mark local-only + if !strings.Contains(refString, "/") { + refString = fmt.Sprintf("localhost/%s", refString) + } + + refAndTags := strings.Split(refString, ",") + baseRef, err := registry.ParseReference(refAndTags[0]) + if err != nil { + return nil, nil, err + } + return &baseRef, refAndTags[1:], nil +} + +func RepoPath(storagePath string, ref *registry.Reference) string { + return filepath.Join(storagePath, ref.Registry, ref.Repository) +} + func GetManifestAndConfig(ctx context.Context, store content.Storage, manifestDesc ocispec.Descriptor) (*ocispec.Manifest, *artifact.KitFile, error) { manifest, err := GetManifest(ctx, store, manifestDesc) if err != nil { @@ -50,3 +80,10 @@ func GetConfig(ctx context.Context, store content.Storage, configDesc ocispec.De } return config, nil } + +func ValidateTag(tag string) error { + if !validTagRegex.MatchString(tag) { + return fmt.Errorf("invalid tag") + } + return nil +} diff --git a/pkg/lib/storage/util_test.go b/pkg/lib/repo/repo_test.go similarity index 96% rename from pkg/lib/storage/util_test.go rename to pkg/lib/repo/repo_test.go index fce3d09a..d1e294f2 100644 --- a/pkg/lib/storage/util_test.go +++ b/pkg/lib/repo/repo_test.go @@ -1,4 +1,4 @@ -package storage +package repo import ( "testing" @@ -7,7 +7,7 @@ import ( "oras.land/oras-go/v2/registry" ) -func TestStringToReference(t *testing.T) { +func TestParseReference(t *testing.T) { tests := []struct { input string expectedRef *registry.Reference diff --git a/pkg/lib/storage/common.go b/pkg/lib/storage/common.go deleted file mode 100644 index 814161ea..00000000 --- a/pkg/lib/storage/common.go +++ /dev/null @@ -1,17 +0,0 @@ -package storage - -import ( - "fmt" - "regexp" -) - -var ( - validTagRegex = regexp.MustCompile(`^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$`) -) - -func validateTag(tag string) error { - if !validTagRegex.MatchString(tag) { - return fmt.Errorf("invalid tag") - } - return nil -} diff --git a/pkg/lib/storage/local.go b/pkg/lib/storage/local.go index d38cf8c1..4bf5b8bd 100644 --- a/pkg/lib/storage/local.go +++ b/pkg/lib/storage/local.go @@ -7,99 +7,38 @@ import ( "fmt" "kitops/pkg/artifact" "kitops/pkg/lib/constants" + "kitops/pkg/lib/repo" "kitops/pkg/output" - "os" - "path/filepath" "github.com/opencontainers/go-digest" specs "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/v2/content" - "oras.land/oras-go/v2/content/oci" + "oras.land/oras-go/v2" ) -type LocalStore struct { - storage *oci.Store - indexPath string - repo string -} - -// Assert LocalStore implements the Store interface. -var _ Store = (*LocalStore)(nil) - -func NewLocalStore(storeRoot, repo string) Store { - storeHome := filepath.Join(storeRoot, repo) - indexPath := filepath.Join(storeHome, "index.json") - - store, err := oci.New(storeHome) - if err != nil { - panic(err) - } - - return &LocalStore{ - storage: store, - indexPath: indexPath, - repo: repo, - } -} - -func (store *LocalStore) SaveModel(ctx context.Context, model *artifact.Model, tag string) (*ocispec.Descriptor, error) { - configDesc, err := store.saveConfigFile(ctx, model.Config) +func SaveModel(ctx context.Context, store oras.Target, model *artifact.Model, tag string) (*ocispec.Descriptor, error) { + configDesc, err := saveConfigFile(ctx, store, model.Config) if err != nil { return nil, err } var layerDescs []ocispec.Descriptor for _, layer := range model.Layers { - layerDesc, err := store.saveContentLayer(ctx, &layer) + layerDesc, err := saveContentLayer(ctx, store, &layer) if err != nil { return nil, err } layerDescs = append(layerDescs, layerDesc) } - manifestDesc, err := store.saveModelManifest(ctx, layerDescs, configDesc, tag) + manifest := CreateManifest(configDesc, layerDescs) + manifestDesc, err := saveModelManifest(ctx, store, manifest, tag) if err != nil { return nil, err } return manifestDesc, nil } -func (store *LocalStore) TagModel(ctx context.Context, manifestDesc ocispec.Descriptor, tag string) error { - if err := validateTag(tag); err != nil { - return err - } - - if err := store.storage.Tag(ctx, manifestDesc, tag); err != nil { - return fmt.Errorf("failed to tag manifest: %w", err) - } - - return nil -} - -func (store *LocalStore) Fetch(ctx context.Context, desc ocispec.Descriptor) ([]byte, error) { - bytes, err := content.FetchAll(ctx, store.storage, desc) - return bytes, err -} - -func (store *LocalStore) ParseIndexJson() (*ocispec.Index, error) { - indexBytes, err := os.ReadFile(store.indexPath) - if err != nil { - return nil, fmt.Errorf("failed to read index: %w", err) - } - - index := &ocispec.Index{} - if err := json.Unmarshal(indexBytes, index); err != nil { - return nil, fmt.Errorf("failed to parse index: %w", err) - } - - return index, nil -} - -func (store *LocalStore) GetRepository() string { - return store.repo -} - -func (store *LocalStore) saveContentLayer(ctx context.Context, layer *artifact.ModelLayer) (ocispec.Descriptor, error) { +func saveContentLayer(ctx context.Context, store oras.Target, layer *artifact.ModelLayer) (ocispec.Descriptor, error) { buf := &bytes.Buffer{} err := layer.Apply(buf) if err != nil { @@ -113,7 +52,7 @@ func (store *LocalStore) saveContentLayer(ctx context.Context, layer *artifact.M Size: int64(buf.Len()), } - exists, err := store.storage.Exists(ctx, desc) + exists, err := store.Exists(ctx, desc) if err != nil { return ocispec.DescriptorEmptyJSON, err } @@ -121,7 +60,7 @@ func (store *LocalStore) saveContentLayer(ctx context.Context, layer *artifact.M output.Infof("Model layer already saved: %s", desc.Digest) } else { // Does not exist in storage, need to push - err = store.storage.Push(ctx, desc, buf) + err = store.Push(ctx, desc, buf) if err != nil { return ocispec.DescriptorEmptyJSON, err } @@ -131,7 +70,7 @@ func (store *LocalStore) saveContentLayer(ctx context.Context, layer *artifact.M return desc, nil } -func (store *LocalStore) saveConfigFile(ctx context.Context, model *artifact.KitFile) (ocispec.Descriptor, error) { +func saveConfigFile(ctx context.Context, store oras.Target, model *artifact.KitFile) (ocispec.Descriptor, error) { modelBytes, err := model.MarshalToJSON() if err != nil { return ocispec.DescriptorEmptyJSON, err @@ -142,13 +81,13 @@ func (store *LocalStore) saveConfigFile(ctx context.Context, model *artifact.Kit Size: int64(len(modelBytes)), } - exists, err := store.storage.Exists(ctx, desc) + exists, err := store.Exists(ctx, desc) if err != nil { return ocispec.DescriptorEmptyJSON, err } if !exists { // Does not exist in storage, need to push - err = store.storage.Push(ctx, desc, bytes.NewReader(modelBytes)) + err = store.Push(ctx, desc, bytes.NewReader(modelBytes)) if err != nil { return ocispec.DescriptorEmptyJSON, err } @@ -160,16 +99,7 @@ func (store *LocalStore) saveConfigFile(ctx context.Context, model *artifact.Kit return desc, nil } -func (store *LocalStore) saveModelManifest(ctx context.Context, layerDescs []ocispec.Descriptor, config ocispec.Descriptor, tag string) (*ocispec.Descriptor, error) { - manifest := ocispec.Manifest{ - Versioned: specs.Versioned{SchemaVersion: 2}, - Config: config, - Layers: []ocispec.Descriptor{}, - Annotations: map[string]string{}, - } - // Add the layers to the manifest - manifest.Layers = append(manifest.Layers, layerDescs...) - +func saveModelManifest(ctx context.Context, store oras.Target, manifest ocispec.Manifest, tag string) (*ocispec.Descriptor, error) { manifestBytes, err := json.Marshal(manifest) if err != nil { return nil, err @@ -181,11 +111,11 @@ func (store *LocalStore) saveModelManifest(ctx context.Context, layerDescs []oci Size: int64(len(manifestBytes)), } - if exists, err := store.storage.Exists(ctx, desc); err != nil { + if exists, err := store.Exists(ctx, desc); err != nil { return nil, err } else if !exists { // Does not exist in storage, need to push - err = store.storage.Push(ctx, desc, bytes.NewReader(manifestBytes)) + err = store.Push(ctx, desc, bytes.NewReader(manifestBytes)) if err != nil { return nil, err } @@ -195,10 +125,10 @@ func (store *LocalStore) saveModelManifest(ctx context.Context, layerDescs []oci } if tag != "" { - if err := validateTag(tag); err != nil { + if err := repo.ValidateTag(tag); err != nil { return nil, err } - if err := store.storage.Tag(ctx, desc, tag); err != nil { + if err := store.Tag(ctx, desc, tag); err != nil { return nil, fmt.Errorf("failed to tag manifest: %w", err) } output.Debugf("Added tag to manifest: %s", tag) @@ -206,3 +136,14 @@ func (store *LocalStore) saveModelManifest(ctx context.Context, layerDescs []oci return &desc, nil } + +func CreateManifest(configDesc ocispec.Descriptor, layerDescs []ocispec.Descriptor) ocispec.Manifest { + manifest := ocispec.Manifest{ + Versioned: specs.Versioned{SchemaVersion: 2}, + Config: configDesc, + Layers: layerDescs, + Annotations: map[string]string{}, + } + + return manifest +} diff --git a/pkg/lib/storage/store.go b/pkg/lib/storage/store.go deleted file mode 100644 index 9f342b90..00000000 --- a/pkg/lib/storage/store.go +++ /dev/null @@ -1,19 +0,0 @@ -package storage - -import ( - "context" - "kitops/pkg/artifact" - - _ "crypto/sha256" - _ "crypto/sha512" - - ocispec "github.com/opencontainers/image-spec/specs-go/v1" -) - -type Store interface { - SaveModel(ctx context.Context, model *artifact.Model, tag string) (*ocispec.Descriptor, error) - TagModel(ctx context.Context, manifestDesc ocispec.Descriptor, tag string) error - GetRepository() string - ParseIndexJson() (*ocispec.Index, error) - Fetch(context.Context, ocispec.Descriptor) ([]byte, error) -} diff --git a/pkg/lib/storage/util.go b/pkg/lib/storage/util.go deleted file mode 100644 index c1afb44a..00000000 --- a/pkg/lib/storage/util.go +++ /dev/null @@ -1,31 +0,0 @@ -package storage - -import ( - "fmt" - "path/filepath" - "strings" - - "oras.land/oras-go/v2/registry" -) - -// ParseReference parses a reference string into a Reference struct. If the -// reference does not include a registry (e.g. myrepo:mytag), the placeholder -// 'localhost' is used. Additional tags can be specified in a comma-separated -// list (e.g. myrepo:tag1,tag2,tag3) -func ParseReference(refString string) (ref *registry.Reference, extraTags []string, err error) { - // References _must_ contain host; use localhost to mark local-only - if !strings.Contains(refString, "/") { - refString = fmt.Sprintf("localhost/%s", refString) - } - - refAndTags := strings.Split(refString, ",") - baseRef, err := registry.ParseReference(refAndTags[0]) - if err != nil { - return nil, nil, err - } - return &baseRef, refAndTags[1:], nil -} - -func LocalStorePath(storageRoot string, ref *registry.Reference) string { - return filepath.Join(storageRoot, ref.Registry, ref.Repository) -} diff --git a/pkg/lib/testing/testing.go b/pkg/lib/testing/testing.go deleted file mode 100644 index fd58477a..00000000 --- a/pkg/lib/testing/testing.go +++ /dev/null @@ -1,85 +0,0 @@ -package internal - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "kitops/pkg/artifact" - "kitops/pkg/lib/storage" - - "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" -) - -const ValidSize = int64(100) -const InvalidSize = int64(-1) - -var TestingNotFoundError = errors.New("artifact not found") -var TestingInvalidSizeError = errors.New("invalid size") - -type TestStore struct { - // Repository for this store - Repo string - // Map of digest to Manifest, to simulate retrieval from e.g. disk - Manifests map[digest.Digest]ocispec.Manifest - // Map of digest to Config, to simulate retrieval from e.g. disk - Configs map[digest.Digest]artifact.KitFile - // Index for the store - Index *ocispec.Index -} - -var _ storage.Store = (*TestStore)(nil) - -// Fetch simulates fetching a blob from the store, given a descriptor. If the object does -// not exist, returns TestingNotFoundError. To simulate mismatched size between the descriptor's -// 'size' field and the size of the object, set the descriptor's size to InvalidSize -func (s *TestStore) Fetch(_ context.Context, desc ocispec.Descriptor) ([]byte, error) { - for digest, manifest := range s.Manifests { - if digest == desc.Digest { - if desc.Size == InvalidSize { - return nil, TestingInvalidSizeError - } - jsonBytes, err := json.Marshal(manifest) - if err != nil { - return nil, fmt.Errorf("testing -- unexpected error: failed to marshal manifest: %w", err) - } - return jsonBytes, nil - } - } - for digest, config := range s.Configs { - if digest == desc.Digest { - if desc.Size == InvalidSize { - return nil, TestingInvalidSizeError - } - jsonBytes, err := json.Marshal(config) - if err != nil { - return nil, fmt.Errorf("testing -- unexpected error: failed to marshal config: %w", err) - } - return jsonBytes, nil - } - } - return nil, TestingNotFoundError -} - -// ParseIndexJson simulates reading the index.json for the store. If an index json does not -// exist, returns TestingNotFoundError -func (s *TestStore) ParseIndexJson() (*ocispec.Index, error) { - if s.Index != nil { - return s.Index, nil - } - return nil, TestingNotFoundError -} - -func (*TestStore) TagModel(context.Context, ocispec.Descriptor, string) error { - return fmt.Errorf("tag model is not implemented for testing") -} - -// SaveModel is not yet implemented! -func (*TestStore) SaveModel(context.Context, *artifact.Model, string) (*ocispec.Descriptor, error) { - return nil, fmt.Errorf("save model is not implemented for testing") -} - -func (t *TestStore) GetRepository() string { - return t.Repo -}