diff --git a/pkg/libmirror/layouts/layouts.go b/pkg/libmirror/layouts/layouts.go index c4b0205a..08057ca1 100644 --- a/pkg/libmirror/layouts/layouts.go +++ b/pkg/libmirror/layouts/layouts.go @@ -270,11 +270,16 @@ func FindDeckhouseModulesImages( modulesData []modules.Module, filter *modules.Filter, ) error { + logger := params.Logger + + counter := 0 for _, module := range modulesData { if !filter.Match(&module) { continue } + counter++ + moduleImageLayouts := layouts.Modules[module.Name] moduleImageLayouts.ReleaseImages = map[string]struct{}{} if filter.ShouldMirrorReleaseChannels(module.Name) { @@ -287,7 +292,10 @@ func FindDeckhouseModulesImages( } } - moduleImages, releaseImages, err := modules.FindExternalModuleImages( + logger.InfoF("%d:\t%s - find external module images", counter, module.Name) + + moduleImages, moduleImagesWithExternal, releaseImages, err := modules.FindExternalModuleImages( + params, &module, filter, params.RegistryAuth, @@ -298,11 +306,14 @@ func FindDeckhouseModulesImages( return fmt.Errorf("Find images of %s: %w", module.Name, err) } - moduleImageLayouts.ModuleImages = moduleImages + moduleImageLayouts.ModuleImages = moduleImagesWithExternal maps.Copy(moduleImageLayouts.ReleaseImages, releaseImages) + logger.InfoF("%d:\t%s - find module extra images", counter, module.Name) + // Find extra images if any exist extraImages, err := modules.FindModuleExtraImages( + params, &module, moduleImages, params.RegistryAuth, @@ -319,6 +330,8 @@ func FindDeckhouseModulesImages( } layouts.Modules[module.Name] = moduleImageLayouts + + logger.InfoF("%d:\t%s", counter, module.Name) } return nil diff --git a/pkg/libmirror/modules/modules.go b/pkg/libmirror/modules/modules.go index 2979edf2..ebe39a5b 100644 --- a/pkg/libmirror/modules/modules.go +++ b/pkg/libmirror/modules/modules.go @@ -22,6 +22,7 @@ import ( "fmt" "io/fs" "path" + "strings" "github.com/Masterminds/semver/v3" "github.com/google/go-containerregistry/pkg/authn" @@ -30,6 +31,7 @@ import ( "github.com/deckhouse/deckhouse-cli/internal/mirror/releases" "github.com/deckhouse/deckhouse-cli/pkg/libmirror/images" + "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/auth" "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/errorutil" ) @@ -101,12 +103,15 @@ func getModulesForRepo( } func FindExternalModuleImages( + params *params.PullParams, mod *Module, filter *Filter, authProvider authn.Authenticator, insecure, skipVerifyTLS bool, -) (moduleImages, releaseImages map[string]struct{}, err error) { - moduleImages, releaseImages = map[string]struct{}{}, map[string]struct{}{} +) (moduleImages []string, moduleImagesWithExternal, releaseImages map[string]struct{}, err error) { + logger := params.Logger + + moduleImagesWithExternal, releaseImages = map[string]struct{}{}, map[string]struct{}{} nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptions(authProvider, insecure, skipVerifyTLS) // Check if specific versions are requested (explicit tags) @@ -127,14 +132,15 @@ func FindExternalModuleImages( if len(versionsToMirror) > 0 && !isDefaultConstraint { // Explicit versions specified (e.g., neuvector@=v1.2.3 or neuvector@~1.2.0) for _, tag := range versionsToMirror { - moduleImages[mod.RegistryPath+":"+tag] = struct{}{} + moduleImages = append(moduleImages, mod.RegistryPath+":"+tag) + moduleImagesWithExternal[mod.RegistryPath+":"+tag] = struct{}{} releaseImages[path.Join(mod.RegistryPath, "release")+":"+tag] = struct{}{} } } else if filter.ShouldMirrorReleaseChannels(mod.Name) { // No explicit versions - use release channels channelImgs, err := getAvailableReleaseChannelsImagesForModule(mod, nameOpts, remoteOpts) if err != nil { - return nil, nil, fmt.Errorf("get release channels: %w", err) + return nil, nil, nil, fmt.Errorf("get release channels: %w", err) } for img := range channelImgs { releaseImages[img] = struct{}{} @@ -142,18 +148,28 @@ func FindExternalModuleImages( channelVers, err := releases.FetchVersionsFromModuleReleaseChannels(channelImgs, authProvider, insecure, skipVerifyTLS) if err != nil { - return nil, nil, fmt.Errorf("fetch channel versions: %w", err) + return nil, nil, nil, fmt.Errorf("fetch channel versions: %w", err) } for _, version := range channelVers { - moduleImages[mod.RegistryPath+":"+version] = struct{}{} + moduleImages = append(moduleImages, mod.RegistryPath+":"+version) + moduleImagesWithExternal[mod.RegistryPath+":"+version] = struct{}{} releaseImages[path.Join(mod.RegistryPath, "release")+":"+version] = struct{}{} } } - for imageTag := range moduleImages { + logger.DebugF("Finding module extra images for %s", mod.Name) + + for _, imageTag := range moduleImages { + if strings.Contains(imageTag, "@sha256:") { + logger.DebugF("Skipping digest reference %s for images_digests.json extraction", imageTag) + continue // Skip digest references + } + + logger.DebugF("Checking module image %s for extra images", imageTag) + ref, err := name.ParseReference(imageTag, nameOpts...) if err != nil { - return nil, nil, fmt.Errorf("Get digests for %q version: %w", imageTag, err) + return nil, nil, nil, fmt.Errorf("Get digests for %q version: %w", imageTag, err) } img, err := remote.Image(ref, remoteOpts...) @@ -161,24 +177,28 @@ func FindExternalModuleImages( if errorutil.IsImageNotFoundError(err) { continue } - return nil, nil, fmt.Errorf("Get digests for %q version: %w", imageTag, err) + return nil, nil, nil, fmt.Errorf("Get digests for %q version: %w", imageTag, err) } + logger.DebugF("Extracting images_digests.json from %s", imageTag) + imagesDigestsJSON, err := images.ExtractFileFromImage(img, "images_digests.json") switch { case errors.Is(err, fs.ErrNotExist): continue case err != nil: - return nil, nil, fmt.Errorf("Extract digests for %q version: %w", imageTag, err) + return nil, nil, nil, fmt.Errorf("Extract digests for %q version: %w", imageTag, err) } + logger.DebugF("Parsing images_digests.json from %s", imageTag) + digests := images.ExtractDigestsFromJSONFile(imagesDigestsJSON.Bytes()) for _, digest := range digests { - moduleImages[mod.RegistryPath+"@"+digest] = struct{}{} + moduleImagesWithExternal[mod.RegistryPath+"@"+digest] = struct{}{} } } - return moduleImages, releaseImages, nil + return moduleImages, moduleImagesWithExternal, releaseImages, nil } func getAvailableReleaseChannelsImagesForModule(mod *Module, refOpts []name.Option, remoteOpts []remote.Option) (map[string]struct{}, error) { @@ -212,16 +232,26 @@ func getAvailableReleaseChannelsImagesForModule(mod *Module, refOpts []name.Opti // FindModuleExtraImages extracts extra_images.json from module images and returns extra images map func FindModuleExtraImages( + params *params.PullParams, mod *Module, - moduleImages map[string]struct{}, + moduleImages []string, authProvider authn.Authenticator, insecure, skipVerifyTLS bool, ) (extraImages map[string]struct{}, err error) { + logger := params.Logger + extraImages = map[string]struct{}{} _, remoteOpts := auth.MakeRemoteRegistryRequestOptions(authProvider, insecure, skipVerifyTLS) // Try to extract extra_images.json from any available module version - for imageTag := range moduleImages { + for _, imageTag := range moduleImages { + if strings.Contains(imageTag, "@sha256:") { + logger.DebugF("Skipping digest reference %s for extra_images.json extraction", imageTag) + continue // Skip digest references + } + + logger.DebugF("Checking module image %s for extra_images.json", imageTag) + ref, err := name.ParseReference(imageTag) if err != nil { continue @@ -232,6 +262,7 @@ func FindModuleExtraImages( continue } + logger.DebugF("Extracting extra_images.json from %s", imageTag) extraImagesJSON, err := images.ExtractFileFromImage(img, "extra_images.json") if errors.Is(err, fs.ErrNotExist) { continue // No extra_images.json in this version, try next diff --git a/pkg/mock/registry_client_mock.go b/pkg/mock/registry_client_mock.go index da56a49f..b598bf13 100644 --- a/pkg/mock/registry_client_mock.go +++ b/pkg/mock/registry_client_mock.go @@ -13,7 +13,6 @@ import ( mm_pkg "github.com/deckhouse/deckhouse-cli/pkg" "github.com/gojuno/minimock/v3" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/remote" ) // RegistryClientMock implements mm_pkg.RegistryClient @@ -28,6 +27,13 @@ type RegistryClientMock struct { beforeExtractImageLayersCounter uint64 ExtractImageLayersMock mRegistryClientMockExtractImageLayers + funcGetDigest func(ctx context.Context, tag string) (hp1 *v1.Hash, err error) + funcGetDigestOrigin string + inspectFuncGetDigest func(ctx context.Context, tag string) + afterGetDigestCounter uint64 + beforeGetDigestCounter uint64 + GetDigestMock mRegistryClientMockGetDigest + funcGetImage func(ctx context.Context, tag string) (i1 v1.Image, err error) funcGetImageOrigin string inspectFuncGetImage func(ctx context.Context, tag string) @@ -56,7 +62,7 @@ type RegistryClientMock struct { beforeGetLabelCounter uint64 GetLabelMock mRegistryClientMockGetLabel - funcGetManifest func(ctx context.Context, tag string) (dp1 *remote.Descriptor, err error) + funcGetManifest func(ctx context.Context, tag string) (ba1 []byte, err error) funcGetManifestOrigin string inspectFuncGetManifest func(ctx context.Context, tag string) afterGetManifestCounter uint64 @@ -110,6 +116,9 @@ func NewRegistryClientMock(t minimock.Tester) *RegistryClientMock { m.ExtractImageLayersMock = mRegistryClientMockExtractImageLayers{mock: m} m.ExtractImageLayersMock.callArgs = []*RegistryClientMockExtractImageLayersParams{} + m.GetDigestMock = mRegistryClientMockGetDigest{mock: m} + m.GetDigestMock.callArgs = []*RegistryClientMockGetDigestParams{} + m.GetImageMock = mRegistryClientMockGetImage{mock: m} m.GetImageMock.callArgs = []*RegistryClientMockGetImageParams{} @@ -517,6 +526,349 @@ func (m *RegistryClientMock) MinimockExtractImageLayersInspect() { } } +type mRegistryClientMockGetDigest struct { + optional bool + mock *RegistryClientMock + defaultExpectation *RegistryClientMockGetDigestExpectation + expectations []*RegistryClientMockGetDigestExpectation + + callArgs []*RegistryClientMockGetDigestParams + mutex sync.RWMutex + + expectedInvocations uint64 + expectedInvocationsOrigin string +} + +// RegistryClientMockGetDigestExpectation specifies expectation struct of the RegistryClient.GetDigest +type RegistryClientMockGetDigestExpectation struct { + mock *RegistryClientMock + params *RegistryClientMockGetDigestParams + paramPtrs *RegistryClientMockGetDigestParamPtrs + expectationOrigins RegistryClientMockGetDigestExpectationOrigins + results *RegistryClientMockGetDigestResults + returnOrigin string + Counter uint64 +} + +// RegistryClientMockGetDigestParams contains parameters of the RegistryClient.GetDigest +type RegistryClientMockGetDigestParams struct { + ctx context.Context + tag string +} + +// RegistryClientMockGetDigestParamPtrs contains pointers to parameters of the RegistryClient.GetDigest +type RegistryClientMockGetDigestParamPtrs struct { + ctx *context.Context + tag *string +} + +// RegistryClientMockGetDigestResults contains results of the RegistryClient.GetDigest +type RegistryClientMockGetDigestResults struct { + hp1 *v1.Hash + err error +} + +// RegistryClientMockGetDigestOrigins contains origins of expectations of the RegistryClient.GetDigest +type RegistryClientMockGetDigestExpectationOrigins struct { + origin string + originCtx string + originTag string +} + +// Marks this method to be optional. The default behavior of any method with Return() is '1 or more', meaning +// the test will fail minimock's automatic final call check if the mocked method was not called at least once. +// Optional() makes method check to work in '0 or more' mode. +// It is NOT RECOMMENDED to use this option unless you really need it, as default behaviour helps to +// catch the problems when the expected method call is totally skipped during test run. +func (mmGetDigest *mRegistryClientMockGetDigest) Optional() *mRegistryClientMockGetDigest { + mmGetDigest.optional = true + return mmGetDigest +} + +// Expect sets up expected params for RegistryClient.GetDigest +func (mmGetDigest *mRegistryClientMockGetDigest) Expect(ctx context.Context, tag string) *mRegistryClientMockGetDigest { + if mmGetDigest.mock.funcGetDigest != nil { + mmGetDigest.mock.t.Fatalf("RegistryClientMock.GetDigest mock is already set by Set") + } + + if mmGetDigest.defaultExpectation == nil { + mmGetDigest.defaultExpectation = &RegistryClientMockGetDigestExpectation{} + } + + if mmGetDigest.defaultExpectation.paramPtrs != nil { + mmGetDigest.mock.t.Fatalf("RegistryClientMock.GetDigest mock is already set by ExpectParams functions") + } + + mmGetDigest.defaultExpectation.params = &RegistryClientMockGetDigestParams{ctx, tag} + mmGetDigest.defaultExpectation.expectationOrigins.origin = minimock.CallerInfo(1) + for _, e := range mmGetDigest.expectations { + if minimock.Equal(e.params, mmGetDigest.defaultExpectation.params) { + mmGetDigest.mock.t.Fatalf("Expectation set by When has same params: %#v", *mmGetDigest.defaultExpectation.params) + } + } + + return mmGetDigest +} + +// ExpectCtxParam1 sets up expected param ctx for RegistryClient.GetDigest +func (mmGetDigest *mRegistryClientMockGetDigest) ExpectCtxParam1(ctx context.Context) *mRegistryClientMockGetDigest { + if mmGetDigest.mock.funcGetDigest != nil { + mmGetDigest.mock.t.Fatalf("RegistryClientMock.GetDigest mock is already set by Set") + } + + if mmGetDigest.defaultExpectation == nil { + mmGetDigest.defaultExpectation = &RegistryClientMockGetDigestExpectation{} + } + + if mmGetDigest.defaultExpectation.params != nil { + mmGetDigest.mock.t.Fatalf("RegistryClientMock.GetDigest mock is already set by Expect") + } + + if mmGetDigest.defaultExpectation.paramPtrs == nil { + mmGetDigest.defaultExpectation.paramPtrs = &RegistryClientMockGetDigestParamPtrs{} + } + mmGetDigest.defaultExpectation.paramPtrs.ctx = &ctx + mmGetDigest.defaultExpectation.expectationOrigins.originCtx = minimock.CallerInfo(1) + + return mmGetDigest +} + +// ExpectTagParam2 sets up expected param tag for RegistryClient.GetDigest +func (mmGetDigest *mRegistryClientMockGetDigest) ExpectTagParam2(tag string) *mRegistryClientMockGetDigest { + if mmGetDigest.mock.funcGetDigest != nil { + mmGetDigest.mock.t.Fatalf("RegistryClientMock.GetDigest mock is already set by Set") + } + + if mmGetDigest.defaultExpectation == nil { + mmGetDigest.defaultExpectation = &RegistryClientMockGetDigestExpectation{} + } + + if mmGetDigest.defaultExpectation.params != nil { + mmGetDigest.mock.t.Fatalf("RegistryClientMock.GetDigest mock is already set by Expect") + } + + if mmGetDigest.defaultExpectation.paramPtrs == nil { + mmGetDigest.defaultExpectation.paramPtrs = &RegistryClientMockGetDigestParamPtrs{} + } + mmGetDigest.defaultExpectation.paramPtrs.tag = &tag + mmGetDigest.defaultExpectation.expectationOrigins.originTag = minimock.CallerInfo(1) + + return mmGetDigest +} + +// Inspect accepts an inspector function that has same arguments as the RegistryClient.GetDigest +func (mmGetDigest *mRegistryClientMockGetDigest) Inspect(f func(ctx context.Context, tag string)) *mRegistryClientMockGetDigest { + if mmGetDigest.mock.inspectFuncGetDigest != nil { + mmGetDigest.mock.t.Fatalf("Inspect function is already set for RegistryClientMock.GetDigest") + } + + mmGetDigest.mock.inspectFuncGetDigest = f + + return mmGetDigest +} + +// Return sets up results that will be returned by RegistryClient.GetDigest +func (mmGetDigest *mRegistryClientMockGetDigest) Return(hp1 *v1.Hash, err error) *RegistryClientMock { + if mmGetDigest.mock.funcGetDigest != nil { + mmGetDigest.mock.t.Fatalf("RegistryClientMock.GetDigest mock is already set by Set") + } + + if mmGetDigest.defaultExpectation == nil { + mmGetDigest.defaultExpectation = &RegistryClientMockGetDigestExpectation{mock: mmGetDigest.mock} + } + mmGetDigest.defaultExpectation.results = &RegistryClientMockGetDigestResults{hp1, err} + mmGetDigest.defaultExpectation.returnOrigin = minimock.CallerInfo(1) + return mmGetDigest.mock +} + +// Set uses given function f to mock the RegistryClient.GetDigest method +func (mmGetDigest *mRegistryClientMockGetDigest) Set(f func(ctx context.Context, tag string) (hp1 *v1.Hash, err error)) *RegistryClientMock { + if mmGetDigest.defaultExpectation != nil { + mmGetDigest.mock.t.Fatalf("Default expectation is already set for the RegistryClient.GetDigest method") + } + + if len(mmGetDigest.expectations) > 0 { + mmGetDigest.mock.t.Fatalf("Some expectations are already set for the RegistryClient.GetDigest method") + } + + mmGetDigest.mock.funcGetDigest = f + mmGetDigest.mock.funcGetDigestOrigin = minimock.CallerInfo(1) + return mmGetDigest.mock +} + +// When sets expectation for the RegistryClient.GetDigest which will trigger the result defined by the following +// Then helper +func (mmGetDigest *mRegistryClientMockGetDigest) When(ctx context.Context, tag string) *RegistryClientMockGetDigestExpectation { + if mmGetDigest.mock.funcGetDigest != nil { + mmGetDigest.mock.t.Fatalf("RegistryClientMock.GetDigest mock is already set by Set") + } + + expectation := &RegistryClientMockGetDigestExpectation{ + mock: mmGetDigest.mock, + params: &RegistryClientMockGetDigestParams{ctx, tag}, + expectationOrigins: RegistryClientMockGetDigestExpectationOrigins{origin: minimock.CallerInfo(1)}, + } + mmGetDigest.expectations = append(mmGetDigest.expectations, expectation) + return expectation +} + +// Then sets up RegistryClient.GetDigest return parameters for the expectation previously defined by the When method +func (e *RegistryClientMockGetDigestExpectation) Then(hp1 *v1.Hash, err error) *RegistryClientMock { + e.results = &RegistryClientMockGetDigestResults{hp1, err} + return e.mock +} + +// Times sets number of times RegistryClient.GetDigest should be invoked +func (mmGetDigest *mRegistryClientMockGetDigest) Times(n uint64) *mRegistryClientMockGetDigest { + if n == 0 { + mmGetDigest.mock.t.Fatalf("Times of RegistryClientMock.GetDigest mock can not be zero") + } + mm_atomic.StoreUint64(&mmGetDigest.expectedInvocations, n) + mmGetDigest.expectedInvocationsOrigin = minimock.CallerInfo(1) + return mmGetDigest +} + +func (mmGetDigest *mRegistryClientMockGetDigest) invocationsDone() bool { + if len(mmGetDigest.expectations) == 0 && mmGetDigest.defaultExpectation == nil && mmGetDigest.mock.funcGetDigest == nil { + return true + } + + totalInvocations := mm_atomic.LoadUint64(&mmGetDigest.mock.afterGetDigestCounter) + expectedInvocations := mm_atomic.LoadUint64(&mmGetDigest.expectedInvocations) + + return totalInvocations > 0 && (expectedInvocations == 0 || expectedInvocations == totalInvocations) +} + +// GetDigest implements mm_pkg.RegistryClient +func (mmGetDigest *RegistryClientMock) GetDigest(ctx context.Context, tag string) (hp1 *v1.Hash, err error) { + mm_atomic.AddUint64(&mmGetDigest.beforeGetDigestCounter, 1) + defer mm_atomic.AddUint64(&mmGetDigest.afterGetDigestCounter, 1) + + mmGetDigest.t.Helper() + + if mmGetDigest.inspectFuncGetDigest != nil { + mmGetDigest.inspectFuncGetDigest(ctx, tag) + } + + mm_params := RegistryClientMockGetDigestParams{ctx, tag} + + // Record call args + mmGetDigest.GetDigestMock.mutex.Lock() + mmGetDigest.GetDigestMock.callArgs = append(mmGetDigest.GetDigestMock.callArgs, &mm_params) + mmGetDigest.GetDigestMock.mutex.Unlock() + + for _, e := range mmGetDigest.GetDigestMock.expectations { + if minimock.Equal(*e.params, mm_params) { + mm_atomic.AddUint64(&e.Counter, 1) + return e.results.hp1, e.results.err + } + } + + if mmGetDigest.GetDigestMock.defaultExpectation != nil { + mm_atomic.AddUint64(&mmGetDigest.GetDigestMock.defaultExpectation.Counter, 1) + mm_want := mmGetDigest.GetDigestMock.defaultExpectation.params + mm_want_ptrs := mmGetDigest.GetDigestMock.defaultExpectation.paramPtrs + + mm_got := RegistryClientMockGetDigestParams{ctx, tag} + + if mm_want_ptrs != nil { + + if mm_want_ptrs.ctx != nil && !minimock.Equal(*mm_want_ptrs.ctx, mm_got.ctx) { + mmGetDigest.t.Errorf("RegistryClientMock.GetDigest got unexpected parameter ctx, expected at\n%s:\nwant: %#v\n got: %#v%s\n", + mmGetDigest.GetDigestMock.defaultExpectation.expectationOrigins.originCtx, *mm_want_ptrs.ctx, mm_got.ctx, minimock.Diff(*mm_want_ptrs.ctx, mm_got.ctx)) + } + + if mm_want_ptrs.tag != nil && !minimock.Equal(*mm_want_ptrs.tag, mm_got.tag) { + mmGetDigest.t.Errorf("RegistryClientMock.GetDigest got unexpected parameter tag, expected at\n%s:\nwant: %#v\n got: %#v%s\n", + mmGetDigest.GetDigestMock.defaultExpectation.expectationOrigins.originTag, *mm_want_ptrs.tag, mm_got.tag, minimock.Diff(*mm_want_ptrs.tag, mm_got.tag)) + } + + } else if mm_want != nil && !minimock.Equal(*mm_want, mm_got) { + mmGetDigest.t.Errorf("RegistryClientMock.GetDigest got unexpected parameters, expected at\n%s:\nwant: %#v\n got: %#v%s\n", + mmGetDigest.GetDigestMock.defaultExpectation.expectationOrigins.origin, *mm_want, mm_got, minimock.Diff(*mm_want, mm_got)) + } + + mm_results := mmGetDigest.GetDigestMock.defaultExpectation.results + if mm_results == nil { + mmGetDigest.t.Fatal("No results are set for the RegistryClientMock.GetDigest") + } + return (*mm_results).hp1, (*mm_results).err + } + if mmGetDigest.funcGetDigest != nil { + return mmGetDigest.funcGetDigest(ctx, tag) + } + mmGetDigest.t.Fatalf("Unexpected call to RegistryClientMock.GetDigest. %v %v", ctx, tag) + return +} + +// GetDigestAfterCounter returns a count of finished RegistryClientMock.GetDigest invocations +func (mmGetDigest *RegistryClientMock) GetDigestAfterCounter() uint64 { + return mm_atomic.LoadUint64(&mmGetDigest.afterGetDigestCounter) +} + +// GetDigestBeforeCounter returns a count of RegistryClientMock.GetDigest invocations +func (mmGetDigest *RegistryClientMock) GetDigestBeforeCounter() uint64 { + return mm_atomic.LoadUint64(&mmGetDigest.beforeGetDigestCounter) +} + +// Calls returns a list of arguments used in each call to RegistryClientMock.GetDigest. +// The list is in the same order as the calls were made (i.e. recent calls have a higher index) +func (mmGetDigest *mRegistryClientMockGetDigest) Calls() []*RegistryClientMockGetDigestParams { + mmGetDigest.mutex.RLock() + + argCopy := make([]*RegistryClientMockGetDigestParams, len(mmGetDigest.callArgs)) + copy(argCopy, mmGetDigest.callArgs) + + mmGetDigest.mutex.RUnlock() + + return argCopy +} + +// MinimockGetDigestDone returns true if the count of the GetDigest invocations corresponds +// the number of defined expectations +func (m *RegistryClientMock) MinimockGetDigestDone() bool { + if m.GetDigestMock.optional { + // Optional methods provide '0 or more' call count restriction. + return true + } + + for _, e := range m.GetDigestMock.expectations { + if mm_atomic.LoadUint64(&e.Counter) < 1 { + return false + } + } + + return m.GetDigestMock.invocationsDone() +} + +// MinimockGetDigestInspect logs each unmet expectation +func (m *RegistryClientMock) MinimockGetDigestInspect() { + for _, e := range m.GetDigestMock.expectations { + if mm_atomic.LoadUint64(&e.Counter) < 1 { + m.t.Errorf("Expected call to RegistryClientMock.GetDigest at\n%s with params: %#v", e.expectationOrigins.origin, *e.params) + } + } + + afterGetDigestCounter := mm_atomic.LoadUint64(&m.afterGetDigestCounter) + // if default expectation was set then invocations count should be greater than zero + if m.GetDigestMock.defaultExpectation != nil && afterGetDigestCounter < 1 { + if m.GetDigestMock.defaultExpectation.params == nil { + m.t.Errorf("Expected call to RegistryClientMock.GetDigest at\n%s", m.GetDigestMock.defaultExpectation.returnOrigin) + } else { + m.t.Errorf("Expected call to RegistryClientMock.GetDigest at\n%s with params: %#v", m.GetDigestMock.defaultExpectation.expectationOrigins.origin, *m.GetDigestMock.defaultExpectation.params) + } + } + // if func was set then invocations count should be greater than zero + if m.funcGetDigest != nil && afterGetDigestCounter < 1 { + m.t.Errorf("Expected call to RegistryClientMock.GetDigest at\n%s", m.funcGetDigestOrigin) + } + + if !m.GetDigestMock.invocationsDone() && afterGetDigestCounter > 0 { + m.t.Errorf("Expected %d calls to RegistryClientMock.GetDigest at\n%s but found %d calls", + mm_atomic.LoadUint64(&m.GetDigestMock.expectedInvocations), m.GetDigestMock.expectedInvocationsOrigin, afterGetDigestCounter) + } +} + type mRegistryClientMockGetImage struct { optional bool mock *RegistryClientMock @@ -1959,7 +2311,7 @@ type RegistryClientMockGetManifestParamPtrs struct { // RegistryClientMockGetManifestResults contains results of the RegistryClient.GetManifest type RegistryClientMockGetManifestResults struct { - dp1 *remote.Descriptor + ba1 []byte err error } @@ -2063,7 +2415,7 @@ func (mmGetManifest *mRegistryClientMockGetManifest) Inspect(f func(ctx context. } // Return sets up results that will be returned by RegistryClient.GetManifest -func (mmGetManifest *mRegistryClientMockGetManifest) Return(dp1 *remote.Descriptor, err error) *RegistryClientMock { +func (mmGetManifest *mRegistryClientMockGetManifest) Return(ba1 []byte, err error) *RegistryClientMock { if mmGetManifest.mock.funcGetManifest != nil { mmGetManifest.mock.t.Fatalf("RegistryClientMock.GetManifest mock is already set by Set") } @@ -2071,13 +2423,13 @@ func (mmGetManifest *mRegistryClientMockGetManifest) Return(dp1 *remote.Descript if mmGetManifest.defaultExpectation == nil { mmGetManifest.defaultExpectation = &RegistryClientMockGetManifestExpectation{mock: mmGetManifest.mock} } - mmGetManifest.defaultExpectation.results = &RegistryClientMockGetManifestResults{dp1, err} + mmGetManifest.defaultExpectation.results = &RegistryClientMockGetManifestResults{ba1, err} mmGetManifest.defaultExpectation.returnOrigin = minimock.CallerInfo(1) return mmGetManifest.mock } // Set uses given function f to mock the RegistryClient.GetManifest method -func (mmGetManifest *mRegistryClientMockGetManifest) Set(f func(ctx context.Context, tag string) (dp1 *remote.Descriptor, err error)) *RegistryClientMock { +func (mmGetManifest *mRegistryClientMockGetManifest) Set(f func(ctx context.Context, tag string) (ba1 []byte, err error)) *RegistryClientMock { if mmGetManifest.defaultExpectation != nil { mmGetManifest.mock.t.Fatalf("Default expectation is already set for the RegistryClient.GetManifest method") } @@ -2108,8 +2460,8 @@ func (mmGetManifest *mRegistryClientMockGetManifest) When(ctx context.Context, t } // Then sets up RegistryClient.GetManifest return parameters for the expectation previously defined by the When method -func (e *RegistryClientMockGetManifestExpectation) Then(dp1 *remote.Descriptor, err error) *RegistryClientMock { - e.results = &RegistryClientMockGetManifestResults{dp1, err} +func (e *RegistryClientMockGetManifestExpectation) Then(ba1 []byte, err error) *RegistryClientMock { + e.results = &RegistryClientMockGetManifestResults{ba1, err} return e.mock } @@ -2135,7 +2487,7 @@ func (mmGetManifest *mRegistryClientMockGetManifest) invocationsDone() bool { } // GetManifest implements mm_pkg.RegistryClient -func (mmGetManifest *RegistryClientMock) GetManifest(ctx context.Context, tag string) (dp1 *remote.Descriptor, err error) { +func (mmGetManifest *RegistryClientMock) GetManifest(ctx context.Context, tag string) (ba1 []byte, err error) { mm_atomic.AddUint64(&mmGetManifest.beforeGetManifestCounter, 1) defer mm_atomic.AddUint64(&mmGetManifest.afterGetManifestCounter, 1) @@ -2155,7 +2507,7 @@ func (mmGetManifest *RegistryClientMock) GetManifest(ctx context.Context, tag st for _, e := range mmGetManifest.GetManifestMock.expectations { if minimock.Equal(*e.params, mm_params) { mm_atomic.AddUint64(&e.Counter, 1) - return e.results.dp1, e.results.err + return e.results.ba1, e.results.err } } @@ -2187,7 +2539,7 @@ func (mmGetManifest *RegistryClientMock) GetManifest(ctx context.Context, tag st if mm_results == nil { mmGetManifest.t.Fatal("No results are set for the RegistryClientMock.GetManifest") } - return (*mm_results).dp1, (*mm_results).err + return (*mm_results).ba1, (*mm_results).err } if mmGetManifest.funcGetManifest != nil { return mmGetManifest.funcGetManifest(ctx, tag) @@ -3764,6 +4116,8 @@ func (m *RegistryClientMock) MinimockFinish() { if !m.minimockDone() { m.MinimockExtractImageLayersInspect() + m.MinimockGetDigestInspect() + m.MinimockGetImageInspect() m.MinimockGetImageConfigInspect() @@ -3807,6 +4161,7 @@ func (m *RegistryClientMock) minimockDone() bool { done := true return done && m.MinimockExtractImageLayersDone() && + m.MinimockGetDigestDone() && m.MinimockGetImageDone() && m.MinimockGetImageConfigDone() && m.MinimockGetImageLayersDone() && diff --git a/pkg/registry.go b/pkg/registry.go index e6e2c7cb..64e90dbe 100644 --- a/pkg/registry.go +++ b/pkg/registry.go @@ -21,7 +21,6 @@ import ( "io" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/remote" ) // RegistryClient defines the contract for interacting with container registries @@ -33,11 +32,17 @@ type RegistryClient interface { // GetRegistry returns the full registry path (host + scope) GetRegistry() string + // GetDigest retrieves the digest for a specific image tag + // The repository is determined by the chained WithScope() calls + GetDigest(ctx context.Context, tag string) (*v1.Hash, error) + // GetManifest retrieves the manifest for a specific image tag // The repository is determined by the chained WithScope() calls - GetManifest(ctx context.Context, tag string) (*remote.Descriptor, error) + GetManifest(ctx context.Context, tag string) ([]byte, error) - // GetImage retrieves an image for a specific reference + // GetImage retrieves an remote image for a specific reference + // Do not return remote image to avoid drop connection with context cancelation. + // It will be in use while passed context will be alive. // The repository is determined by the chained WithScope() calls GetImage(ctx context.Context, tag string) (v1.Image, error) diff --git a/pkg/registry/client.go b/pkg/registry/client.go index ce68b17d..c6fb6ef0 100644 --- a/pkg/registry/client.go +++ b/pkg/registry/client.go @@ -89,9 +89,42 @@ func (c *Client) GetRegistry() string { return fmt.Sprintf("%s/%s", c.registryHost, c.scopePath) } +// The repository is determined by the chained WithScope() calls +func (c *Client) GetDigest(ctx context.Context, tag string) (*v1.Hash, error) { + fullRegistry := c.GetRegistry() + logentry := c.log.With( + slog.String("registry_host", c.registryHost), + slog.String("scope", c.scopePath), + slog.String("tag", tag), + ) + + logentry.Debug("Getting manifest") + + ref, err := name.ParseReference(fmt.Sprintf("%s:%s", fullRegistry, tag)) + if err != nil { + return nil, fmt.Errorf("failed to parse reference: %w", err) + } + + opts := append(c.options, remote.WithContext(ctx)) + + head, err := remote.Head(ref, opts...) + if err == nil { + return &head.Digest, nil + } + + desc, err := remote.Get(ref, opts...) + if err != nil { + return nil, fmt.Errorf("failed to get manifest: %w", err) + } + + logentry.Debug("Manifest retrieved successfully") + + return &desc.Digest, nil +} + // GetManifest retrieves the manifest for a specific image tag // The repository is determined by the chained WithScope() calls -func (c *Client) GetManifest(ctx context.Context, tag string) (*remote.Descriptor, error) { +func (c *Client) GetManifest(ctx context.Context, tag string) ([]byte, error) { fullRegistry := c.GetRegistry() logentry := c.log.With( slog.String("registry_host", c.registryHost), @@ -114,7 +147,7 @@ func (c *Client) GetManifest(ctx context.Context, tag string) (*remote.Descripto logentry.Debug("Manifest retrieved successfully") - return desc, nil + return desc.Manifest, nil } // GetImage retrieves an remote image for a specific reference