Skip to content

Commit

Permalink
CLID-17: feat: docker v2s2 manifest list (fat manifest) / oci image i…
Browse files Browse the repository at this point in the history
…ndex implementation
  • Loading branch information
aguidirh committed Mar 14, 2024
1 parent c4f8295 commit 6e091bd
Show file tree
Hide file tree
Showing 21 changed files with 270 additions and 128 deletions.
1 change: 0 additions & 1 deletion v2/pkg/additional/const.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,4 @@ package additional
const (
dockerProtocol string = "docker://"
ociProtocol string = "oci://"
hashTruncLen int = 12
)
6 changes: 3 additions & 3 deletions v2/pkg/additional/local_stored_collector.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func (o LocalStorageCollector) AdditionalImagesCollector(ctx context.Context) ([
src = imgSpec.ReferenceWithTransport

if imgSpec.IsImageByDigest() {
dest = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, imgSpec.PathComponent + ":" + imgSpec.Digest[:hashTruncLen]}, "/")
dest = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, imgSpec.PathComponent + ":" + imgSpec.Digest}, "/")
} else {
dest = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, imgSpec.PathComponent}, "/") + ":" + imgSpec.Tag
}
Expand All @@ -66,8 +66,8 @@ func (o LocalStorageCollector) AdditionalImagesCollector(ctx context.Context) ([
}

if imgSpec.IsImageByDigest() {
src = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, imgSpec.PathComponent + ":" + imgSpec.Digest[:hashTruncLen]}, "/")
dest = strings.Join([]string{o.Opts.Destination, imgSpec.PathComponent + ":" + imgSpec.Digest[:hashTruncLen]}, "/")
src = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, imgSpec.PathComponent + ":" + imgSpec.Digest}, "/")
dest = strings.Join([]string{o.Opts.Destination, imgSpec.PathComponent + ":" + imgSpec.Digest}, "/")
} else {
src = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, imgSpec.PathComponent}, "/") + ":" + imgSpec.Tag
dest = strings.Join([]string{o.Opts.Destination, imgSpec.PathComponent}, "/") + ":" + imgSpec.Tag
Expand Down
4 changes: 4 additions & 0 deletions v2/pkg/additional/local_stored_collector_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -186,3 +186,7 @@ func (o MockManifest) ExtractLayersOCI(filePath, toPath, label string, oci *v1al
func (o MockManifest) ExtractLayers(filePath, name, label string) error {
return nil
}

func (o MockManifest) ConvertIndexToSingleManifest(dir string, oci *v1alpha3.OCISchema) error {
return nil
}
12 changes: 6 additions & 6 deletions v2/pkg/api/v1alpha3/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,16 +82,16 @@ type Status struct {
// OCISchema
type OCISchema struct {
SchemaVersion int `json:"schemaVersion"`
MediaType string `json:"mediaType"`
MediaType string `json:"mediaType,omitempty"`
Manifests []OCIManifest `json:"manifests"`
Config OCIManifest `json:"config"`
Layers []OCIManifest `json:"layers"`
Config OCIManifest `json:"config,omitempty"`
Layers []OCIManifest `json:"layers,omitempty"`
}

type OCIManifest struct {
MediaType string `json:"mediaType"`
Digest string `json:"digest"`
Size int `json:"size"`
MediaType string `json:"mediaType,omitempty"`
Digest string `json:"digest,omitempty"`
Size int `json:"size,omitempty"`
}

// OperatorConfigSchema
Expand Down
1 change: 0 additions & 1 deletion v2/pkg/batch/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,6 @@ func (o *Batch) Worker(ctx context.Context, images []v1alpha3.CopyImageSchema, o
index := (i * b.BatchSize) + x
o.Log.Debug("source %s ", images[index].Source)
o.Log.Debug("destination %s ", images[index].Destination)
opts.MultiArch = "all"
go func(ctx context.Context, src, dest string, opts *mirror.CopyOptions, writer bufio.Writer) {
defer wg.Done()
err := o.Mirror.Run(ctx, src, dest, "copy", opts, writer)
Expand Down
22 changes: 13 additions & 9 deletions v2/pkg/batch/worker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,22 +65,22 @@ func TestWorker(t *testing.T) {
type Mirror struct{}
type Manifest struct{}

func (o *Mirror) Run(ctx context.Context, src, dest string, mode mirror.Mode, opts *mirror.CopyOptions, stdout bufio.Writer) error {
func (o Mirror) Run(ctx context.Context, src, dest string, mode mirror.Mode, opts *mirror.CopyOptions, stdout bufio.Writer) error {
return nil
}

func (o *Mirror) Check(ctx context.Context, image string, opts *mirror.CopyOptions) (bool, error) {
func (o Mirror) Check(ctx context.Context, image string, opts *mirror.CopyOptions) (bool, error) {
return true, nil
}

func (o *Manifest) GetOperatorConfig(file string) (*v1alpha3.OperatorConfigSchema, error) {
func (o Manifest) GetOperatorConfig(file string) (*v1alpha3.OperatorConfigSchema, error) {
opcl := v1alpha3.OperatorLabels{OperatorsOperatorframeworkIoIndexConfigsV1: "/configs"}
opc := v1alpha3.OperatorConfig{Labels: opcl}
ocs := &v1alpha3.OperatorConfigSchema{Config: opc}
return ocs, nil
}

func (o *Manifest) GetReleaseSchema(filePath string) ([]v1alpha3.RelatedImage, error) {
func (o Manifest) GetReleaseSchema(filePath string) ([]v1alpha3.RelatedImage, error) {
relatedImages := []v1alpha3.RelatedImage{
{Name: "testA", Image: "registry/name/namespace/sometestimage-a@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea"},
{Name: "testB", Image: "registry/name/namespace/sometestimage-b@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea"},
Expand All @@ -90,7 +90,7 @@ func (o *Manifest) GetReleaseSchema(filePath string) ([]v1alpha3.RelatedImage, e
return relatedImages, nil
}

func (o *Manifest) GetImageIndex(name string) (*v1alpha3.OCISchema, error) {
func (o Manifest) GetImageIndex(name string) (*v1alpha3.OCISchema, error) {
return &v1alpha3.OCISchema{
SchemaVersion: 2,
Manifests: []v1alpha3.OCIManifest{
Expand All @@ -103,7 +103,7 @@ func (o *Manifest) GetImageIndex(name string) (*v1alpha3.OCISchema, error) {
}, nil
}

func (o *Manifest) GetImageManifest(name string) (*v1alpha3.OCISchema, error) {
func (o Manifest) GetImageManifest(name string) (*v1alpha3.OCISchema, error) {
return &v1alpha3.OCISchema{
SchemaVersion: 2,
Manifests: []v1alpha3.OCIManifest{
Expand All @@ -125,7 +125,7 @@ func (o Manifest) GetCatalog(filePath string) (manifest.OperatorCatalog, error)
return manifest.OperatorCatalog{}, nil
}

func (o *Manifest) GetRelatedImagesFromCatalog(operatorCatalog manifest.OperatorCatalog, ctlgInIsc v1alpha2.Operator) (map[string][]v1alpha3.RelatedImage, error) {
func (o Manifest) GetRelatedImagesFromCatalog(operatorCatalog manifest.OperatorCatalog, ctlgInIsc v1alpha2.Operator) (map[string][]v1alpha3.RelatedImage, error) {
relatedImages := make(map[string][]v1alpha3.RelatedImage)
relatedImages["abc"] = []v1alpha3.RelatedImage{
{Name: "testA", Image: "sometestimage-a@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea"},
Expand All @@ -134,10 +134,14 @@ func (o *Manifest) GetRelatedImagesFromCatalog(operatorCatalog manifest.Operator
return relatedImages, nil
}

func (o *Manifest) ExtractLayersOCI(filePath, toPath, label string, oci *v1alpha3.OCISchema) error {
func (o Manifest) ExtractLayersOCI(filePath, toPath, label string, oci *v1alpha3.OCISchema) error {
return nil
}

func (o *Manifest) ExtractLayers(filePath, name, label string) error {
func (o Manifest) ExtractLayers(filePath, name, label string) error {
return nil
}

func (o Manifest) ConvertIndexToSingleManifest(dir string, oci *v1alpha3.OCISchema) error {
return nil
}
42 changes: 20 additions & 22 deletions v2/pkg/cli/executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ type ExecutorSchema struct {
LogsDir string
registryLogFile *os.File
Config v1alpha2.ImageSetConfiguration
Opts mirror.CopyOptions
Opts *mirror.CopyOptions
Operator operator.CollectorInterface
Release release.CollectorInterface
AdditionalImages additional.CollectorInterface
Expand Down Expand Up @@ -126,7 +126,7 @@ func NewMirrorCmd(log clog.PluggableLoggerInterface) *cobra.Command {
flagDestOpts, destOpts := mirror.ImageDestFlags(global, sharedOpts, deprecatedTLSVerifyOpt, "dest-", "dcreds")
flagRetryOpts, retryOpts := mirror.RetryFlags()

opts := mirror.CopyOptions{
opts := &mirror.CopyOptions{
Global: global,
DeprecatedTLSVerify: deprecatedTLSVerifyOpt,
SrcImage: srcOpts,
Expand Down Expand Up @@ -324,24 +324,24 @@ func (o *ExecutorSchema) Complete(args []string) error {

client, _ := release.NewOCPClient(uuid.New())

o.ImageBuilder = imagebuilder.NewBuilder(o.Log, o.Opts)
o.ImageBuilder = imagebuilder.NewBuilder(o.Log, *o.Opts)

signature := release.NewSignatureClient(o.Log, o.Config, o.Opts)
cn := release.NewCincinnati(o.Log, &o.Config, o.Opts, client, false, signature)
o.Release = release.New(o.Log, o.LogsDir, o.Config, o.Opts, o.Mirror, o.Manifest, cn, o.LocalStorageFQDN, o.ImageBuilder)
o.Operator = operator.New(o.Log, o.LogsDir, o.Config, o.Opts, o.Mirror, o.Manifest, o.LocalStorageFQDN)
o.AdditionalImages = additional.New(o.Log, o.Config, o.Opts, o.Mirror, o.Manifest, o.LocalStorageFQDN)
signature := release.NewSignatureClient(o.Log, o.Config, *o.Opts)
cn := release.NewCincinnati(o.Log, &o.Config, *o.Opts, client, false, signature)
o.Release = release.New(o.Log, o.LogsDir, o.Config, *o.Opts, o.Mirror, o.Manifest, cn, o.LocalStorageFQDN, o.ImageBuilder)
o.Operator = operator.New(o.Log, o.LogsDir, o.Config, *o.Opts, o.Mirror, o.Manifest, o.LocalStorageFQDN)
o.AdditionalImages = additional.New(o.Log, o.Config, *o.Opts, o.Mirror, o.Manifest, o.LocalStorageFQDN)
o.ClusterResources = clusterresources.New(o.Log, o.Opts.Global.WorkingDir, o.Config)
o.Batch = batch.New(o.Log, o.LogsDir, o.Mirror, o.Manifest)

if o.Opts.IsMirrorToDisk() {
if o.Opts.Global.StrictArchiving {
o.MirrorArchiver, err = archive.NewMirrorArchive(&o.Opts, rootDir, o.Opts.Global.ConfigPath, o.Opts.Global.WorkingDir, o.LocalStorageDisk, o.Config.ImageSetConfigurationSpec.ArchiveSize, o.Log)
o.MirrorArchiver, err = archive.NewMirrorArchive(o.Opts, rootDir, o.Opts.Global.ConfigPath, o.Opts.Global.WorkingDir, o.LocalStorageDisk, o.Config.ImageSetConfigurationSpec.ArchiveSize, o.Log)
if err != nil {
return err
}
} else {
o.MirrorArchiver, err = archive.NewPermissiveMirrorArchive(&o.Opts, rootDir, o.Opts.Global.ConfigPath, o.Opts.Global.WorkingDir, o.LocalStorageDisk, o.Config.ImageSetConfigurationSpec.ArchiveSize, o.Log)
o.MirrorArchiver, err = archive.NewPermissiveMirrorArchive(o.Opts, rootDir, o.Opts.Global.ConfigPath, o.Opts.Global.WorkingDir, o.LocalStorageDisk, o.Config.ImageSetConfigurationSpec.ArchiveSize, o.Log)
if err != nil {
return err
}
Expand All @@ -357,10 +357,8 @@ func (o *ExecutorSchema) Complete(args []string) error {

// Run - start the mirror functionality
func (o *ExecutorSchema) Run(cmd *cobra.Command, args []string) error {

// make sure we always get multi-arch images
o.Opts.MultiArch = "all"
var err error
o.Opts.MultiArch = "all"
if o.Opts.IsMirrorToDisk() {
err = o.RunMirrorToDisk(cmd, args)

Expand Down Expand Up @@ -573,7 +571,7 @@ func (o *ExecutorSchema) RunMirrorToDisk(cmd *cobra.Command, args []string) erro
collectionFinish := time.Now()

//call the batch worker
err = o.Batch.Worker(cmd.Context(), allImages, o.Opts)
err = o.Batch.Worker(cmd.Context(), allImages, *o.Opts)
if err != nil {
return err
}
Expand Down Expand Up @@ -629,7 +627,7 @@ func (o *ExecutorSchema) RunDiskToMirror(cmd *cobra.Command, args []string) erro
}
}
//call the batch worker
err = o.Batch.Worker(cmd.Context(), allImages, o.Opts)
err = o.Batch.Worker(cmd.Context(), allImages, *o.Opts)
if err != nil {
return err
}
Expand Down Expand Up @@ -770,7 +768,7 @@ func NewPrepareCommand(log clog.PluggableLoggerInterface) *cobra.Command {
flagDestOpts, destOpts := mirror.ImageDestFlags(global, sharedOpts, deprecatedTLSVerifyOpt, "dest-", "dcreds")
flagRetryOpts, retryOpts := mirror.RetryFlags()

opts := mirror.CopyOptions{
opts := &mirror.CopyOptions{
Global: global,
DeprecatedTLSVerify: deprecatedTLSVerifyOpt,
SrcImage: srcOpts,
Expand Down Expand Up @@ -882,11 +880,11 @@ func (o *ExecutorSchema) CompletePrepare(args []string) error {
}
client, _ := release.NewOCPClient(uuid.New())

signature := release.NewSignatureClient(o.Log, o.Config, o.Opts)
cn := release.NewCincinnati(o.Log, &o.Config, o.Opts, client, false, signature)
o.Release = release.New(o.Log, o.LogsDir, o.Config, o.Opts, o.Mirror, o.Manifest, cn, o.LocalStorageFQDN, o.ImageBuilder)
o.Operator = operator.New(o.Log, o.LogsDir, o.Config, o.Opts, o.Mirror, o.Manifest, o.LocalStorageFQDN)
o.AdditionalImages = additional.New(o.Log, o.Config, o.Opts, o.Mirror, o.Manifest, o.LocalStorageFQDN)
signature := release.NewSignatureClient(o.Log, o.Config, *o.Opts)
cn := release.NewCincinnati(o.Log, &o.Config, *o.Opts, client, false, signature)
o.Release = release.New(o.Log, o.LogsDir, o.Config, *o.Opts, o.Mirror, o.Manifest, cn, o.LocalStorageFQDN, o.ImageBuilder)
o.Operator = operator.New(o.Log, o.LogsDir, o.Config, *o.Opts, o.Mirror, o.Manifest, o.LocalStorageFQDN)
o.AdditionalImages = additional.New(o.Log, o.Config, *o.Opts, o.Mirror, o.Manifest, o.LocalStorageFQDN)
return nil
}

Expand Down Expand Up @@ -914,7 +912,7 @@ func (o *ExecutorSchema) RunPrepare(cmd *cobra.Command, args []string) error {
var buff bytes.Buffer
for _, img := range allImages {
buff.WriteString(img.Destination + "\n")
exists, err := o.Mirror.Check(cmd.Context(), img.Destination, &o.Opts)
exists, err := o.Mirror.Check(cmd.Context(), img.Destination, o.Opts)
if err != nil {
o.Log.Warn("unable to check existence of %s in local cache: %v", img.Destination, err)
}
Expand Down
38 changes: 19 additions & 19 deletions v2/pkg/cli/executor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func TestExecutorMirroring(t *testing.T) {
_, destOpts := mirror.ImageDestFlags(global, sharedOpts, deprecatedTLSVerifyOpt, "dest-", "dcreds")
_, retryOpts := mirror.RetryFlags()

opts := mirror.CopyOptions{
opts := &mirror.CopyOptions{
Global: global,
DeprecatedTLSVerify: deprecatedTLSVerifyOpt,
SrcImage: srcOpts,
Expand Down Expand Up @@ -85,8 +85,8 @@ func TestExecutorMirroring(t *testing.T) {
nie.Is(fmt.Errorf("interrupt error"))

t.Run("Testing Executor : mirrorToDisk should pass", func(t *testing.T) {
collector := &Collector{Log: log, Config: cfg, Opts: opts, Fail: false}
batch := &Batch{Log: log, Config: cfg, Opts: opts}
collector := &Collector{Log: log, Config: cfg, Opts: *opts, Fail: false}
batch := &Batch{Log: log, Config: cfg, Opts: *opts}
archiver := MockArchiver{opts.Destination}

ex := &ExecutorSchema{
Expand Down Expand Up @@ -116,8 +116,8 @@ func TestExecutorMirroring(t *testing.T) {
})

t.Run("Testing Executor : diskToMirror should pass", func(t *testing.T) {
collector := &Collector{Log: log, Config: cfg, Opts: opts, Fail: false}
batch := &Batch{Log: log, Config: cfg, Opts: opts}
collector := &Collector{Log: log, Config: cfg, Opts: *opts, Fail: false}
batch := &Batch{Log: log, Config: cfg, Opts: *opts}
archiver := MockMirrorUnArchiver{}
cr := MockClusterResources{}
cfg.Mirror.Platform.Graph = true
Expand Down Expand Up @@ -150,8 +150,8 @@ func TestExecutorMirroring(t *testing.T) {
})

t.Run("Testing Executor : diskToMirror should fail", func(t *testing.T) {
collector := &Collector{Log: log, Config: cfg, Opts: opts, Fail: false}
batch := &Batch{Log: log, Config: cfg, Opts: opts}
collector := &Collector{Log: log, Config: cfg, Opts: *opts, Fail: false}
batch := &Batch{Log: log, Config: cfg, Opts: *opts}
archiver := MockMirrorUnArchiver{Fail: true}
cr := MockClusterResources{}
cfg.Mirror.Platform.Graph = true
Expand Down Expand Up @@ -209,7 +209,7 @@ func TestExecutorValidate(t *testing.T) {
_, destOpts := mirror.ImageDestFlags(global, sharedOpts, deprecatedTLSVerifyOpt, "dest-", "dcreds")
_, retryOpts := mirror.RetryFlags()

opts := mirror.CopyOptions{
opts := &mirror.CopyOptions{
Global: global,
DeprecatedTLSVerify: deprecatedTLSVerifyOpt,
SrcImage: srcOpts,
Expand Down Expand Up @@ -288,7 +288,7 @@ func TestExecutorComplete(t *testing.T) {
_, destOpts := mirror.ImageDestFlags(global, sharedOpts, deprecatedTLSVerifyOpt, "dest-", "dcreds")
_, retryOpts := mirror.RetryFlags()

opts := mirror.CopyOptions{
opts := &mirror.CopyOptions{
Global: global,
DeprecatedTLSVerify: deprecatedTLSVerifyOpt,
SrcImage: srcOpts,
Expand Down Expand Up @@ -359,7 +359,7 @@ func TestExecutorValidatePrepare(t *testing.T) {
_, destOpts := mirror.ImageDestFlags(global, sharedOpts, deprecatedTLSVerifyOpt, "dest-", "dcreds")
_, retryOpts := mirror.RetryFlags()

opts := mirror.CopyOptions{
opts := &mirror.CopyOptions{
Global: global,
DeprecatedTLSVerify: deprecatedTLSVerifyOpt,
SrcImage: srcOpts,
Expand Down Expand Up @@ -420,7 +420,7 @@ func TestExecutorCompletePrepare(t *testing.T) {
_, destOpts := mirror.ImageDestFlags(global, sharedOpts, deprecatedTLSVerifyOpt, "dest-", "dcreds")
_, retryOpts := mirror.RetryFlags()

opts := mirror.CopyOptions{
opts := &mirror.CopyOptions{
Global: global,
DeprecatedTLSVerify: deprecatedTLSVerifyOpt,
SrcImage: srcOpts,
Expand Down Expand Up @@ -468,7 +468,7 @@ func TestExecutorRunPrepare(t *testing.T) {
_, destOpts := mirror.ImageDestFlags(global, sharedOpts, deprecatedTLSVerifyOpt, "dest-", "dcreds")
_, retryOpts := mirror.RetryFlags()

opts := mirror.CopyOptions{
opts := &mirror.CopyOptions{
Global: global,
DeprecatedTLSVerify: deprecatedTLSVerifyOpt,
SrcImage: srcOpts,
Expand Down Expand Up @@ -500,7 +500,7 @@ func TestExecutorRunPrepare(t *testing.T) {
log.Error("imagesetconfig %v ", err)
}

collector := &Collector{Log: log, Config: cfg, Opts: opts, Fail: false}
collector := &Collector{Log: log, Config: cfg, Opts: *opts, Fail: false}
mockMirror := Mirror{}

ex := &ExecutorSchema{
Expand Down Expand Up @@ -543,7 +543,7 @@ func TestExecutorSetupLocalStorage(t *testing.T) {
_, destOpts := mirror.ImageDestFlags(global, sharedOpts, deprecatedTLSVerifyOpt, "dest-", "dcreds")
_, retryOpts := mirror.RetryFlags()

opts := mirror.CopyOptions{
opts := &mirror.CopyOptions{
Global: global,
DeprecatedTLSVerify: deprecatedTLSVerifyOpt,
SrcImage: srcOpts,
Expand Down Expand Up @@ -577,7 +577,7 @@ func TestExecutorSetupWorkingDir(t *testing.T) {
WorkingDir: "/root",
}

opts := mirror.CopyOptions{
opts := &mirror.CopyOptions{
Global: global,
}

Expand Down Expand Up @@ -627,7 +627,7 @@ func TestExecutorSetupLogsLevelAndDir(t *testing.T) {
SecurePolicy: false,
}

opts := mirror.CopyOptions{
opts := &mirror.CopyOptions{
Global: global,
}

Expand Down Expand Up @@ -668,7 +668,7 @@ func TestExecutorCollectAll(t *testing.T) {
_, destOpts := mirror.ImageDestFlags(global, sharedOpts, deprecatedTLSVerifyOpt, "dest-", "dcreds")
_, retryOpts := mirror.RetryFlags()

opts := mirror.CopyOptions{
opts := &mirror.CopyOptions{
Global: global,
DeprecatedTLSVerify: deprecatedTLSVerifyOpt,
SrcImage: srcOpts,
Expand All @@ -680,8 +680,8 @@ func TestExecutorCollectAll(t *testing.T) {

// read the ImageSetConfiguration
cfg, _ := config.ReadConfig("../../tests/isc.yaml")
failCollector := &Collector{Log: log, Config: cfg, Opts: opts, Fail: true}
collector := &Collector{Log: log, Config: cfg, Opts: opts, Fail: false}
failCollector := &Collector{Log: log, Config: cfg, Opts: *opts, Fail: true}
collector := &Collector{Log: log, Config: cfg, Opts: *opts, Fail: false}

mkdir := MockMakeDir{}

Expand Down

0 comments on commit 6e091bd

Please sign in to comment.