Skip to content

Commit

Permalink
feat(build): support mirror registries
Browse files Browse the repository at this point in the history
Signed-off-by: Alexey Igrychev <alexey.igrychev@flant.com>
  • Loading branch information
alexey-igrychev committed May 30, 2024
1 parent 9494b91 commit bb741a3
Show file tree
Hide file tree
Showing 6 changed files with 257 additions and 3 deletions.
4 changes: 4 additions & 0 deletions pkg/docker/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ const (
ctxDockerCliKey = "docker_cli"
)

func IsEnabled() bool {
return defaultCLi != nil
}

func Init(ctx context.Context, dockerConfigDir string, verbose, debug bool) error {
if dockerConfigDir != "" {
cliconfig.SetDir(dockerConfigDir)
Expand Down
11 changes: 11 additions & 0 deletions pkg/docker/system.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package docker

import (
"context"

"github.com/docker/docker/api/types"
)

func Info(ctx context.Context) (types.Info, error) {
return apiCli(ctx).Info(ctx)
}
35 changes: 35 additions & 0 deletions pkg/docker_registry/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"strings"
"time"

dockerReference "github.com/docker/distribution/reference"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
Expand Down Expand Up @@ -234,6 +235,40 @@ func (api *api) newRepositoryOptions() []name.Option {
return api.parseReferenceOptions()
}

type referenceParts struct {
registry string
repository string
tag string
digest string
}

func (api *api) parseReferenceParts(reference string) (referenceParts, error) {
parsedReference, err := name.ParseReference(reference, api.parseReferenceOptions()...)
if err != nil {
return referenceParts{}, err
}

// res[0] full match
// res[1] repository
// res[2] tag
// res[3] digest
res := dockerReference.ReferenceRegexp.FindStringSubmatch(reference)
if len(res) != 4 {
panic(fmt.Sprintf("unexpected regexp find submatch result %v for reference %q (%d)", res, reference, len(res)))
}

referenceParts := referenceParts{}
referenceParts.registry = parsedReference.Context().RegistryStr()
referenceParts.repository = parsedReference.Context().RepositoryStr()
referenceParts.tag = res[2]
if referenceParts.tag == "" {
referenceParts.tag = "latest"
}
referenceParts.digest = res[3]

return referenceParts, nil
}

func (api *api) parseReferenceOptions() []name.Option {
var options []name.Option
options = append(options, name.WeakValidation)
Expand Down
60 changes: 60 additions & 0 deletions pkg/docker_registry/api_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package docker_registry

import (
"github.com/google/go-containerregistry/pkg/name"
. "github.com/onsi/ginkgo/extensions/table"
. "github.com/onsi/gomega"
)

type ParseReferencePartsEntry struct {
reference string
expectation referenceParts
}

var _ = DescribeTable("Api_ParseReferenceParts", func(entry ParseReferencePartsEntry) {
parts, err := (&api{}).parseReferenceParts(entry.reference)
Ω(err).ShouldNot(HaveOccurred())
Ω(parts).Should(Equal(entry.expectation))
},
Entry("account/project", ParseReferencePartsEntry{
reference: "account/project",
expectation: referenceParts{
registry: name.DefaultRegistry,
repository: "account/project",
tag: "latest",
},
}),
Entry("repo", ParseReferencePartsEntry{
reference: "repo",
expectation: referenceParts{
registry: name.DefaultRegistry,
repository: "library/repo",
tag: "latest",
},
}),
Entry("registry.com/repo", ParseReferencePartsEntry{
reference: "registry.com/repo",
expectation: referenceParts{
registry: "registry.com",
repository: "repo",
tag: "latest",
},
}),
Entry("registry.com/repo:tag", ParseReferencePartsEntry{
reference: "registry.com/repo:tag",
expectation: referenceParts{
registry: "registry.com",
repository: "repo",
tag: "tag",
},
}),
Entry("registry.com/repo:tag@sha256:db6697a61d5679b7ca69dbde3dad6be0d17064d5b6b0e9f7be8d456ebb337209", ParseReferencePartsEntry{
reference: "registry.com/repo:tag@sha256:db6697a61d5679b7ca69dbde3dad6be0d17064d5b6b0e9f7be8d456ebb337209",
expectation: referenceParts{
registry: "registry.com",
repository: "repo",
tag: "tag",
digest: "sha256:db6697a61d5679b7ca69dbde3dad6be0d17064d5b6b0e9f7be8d456ebb337209",
},
}),
)
140 changes: 140 additions & 0 deletions pkg/docker_registry/generic_api.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
package docker_registry

import (
"context"
"fmt"
"net/url"
"sync"

"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"

"github.com/werf/werf/pkg/docker"
"github.com/werf/werf/pkg/image"
)

type genericApi struct {
commonApi *api
mirrors *[]string
mutex sync.Mutex
}

func newGenericApi(_ context.Context, options apiOptions) (*genericApi, error) {
d := &genericApi{}
d.commonApi = newAPI(options)
return d, nil
}

func (api *genericApi) GetRepoImageConfigFile(ctx context.Context, reference string) (*v1.ConfigFile, error) {
mirrorReferenceList, err := api.mirrorReferenceList(ctx, reference)
if err != nil {
return nil, fmt.Errorf("unable to prepare mirror reference list: %w", err)
}

for _, mirrorReference := range mirrorReferenceList {
config, err := api.getRepoImageConfigFile(ctx, mirrorReference)
if err != nil {
if IsBlobUnknownError(err) || IsManifestUnknownError(err) || IsNameUnknownError(err) {
continue
}

return nil, err
}

return config, nil
}

return api.getRepoImageConfigFile(ctx, reference)
}

func (api *genericApi) getRepoImageConfigFile(_ context.Context, reference string) (*v1.ConfigFile, error) {
imageInfo, _, err := api.commonApi.image(reference)
if err != nil {
return nil, err
}

return imageInfo.ConfigFile()
}

func (api *genericApi) GetRepoImage(ctx context.Context, reference string) (*image.Info, error) {
mirrorReferenceList, err := api.mirrorReferenceList(ctx, reference)
if err != nil {
return nil, fmt.Errorf("unable to prepare mirror reference list: %w", err)
}

for _, mirrorReference := range mirrorReferenceList {
info, err := api.commonApi.TryGetRepoImage(ctx, mirrorReference)
if err != nil {
return nil, fmt.Errorf("unable to try getting mirror repo image %q: %w", mirrorReference, err)
}

if info != nil {
return info, nil
}
}

return api.commonApi.GetRepoImage(ctx, reference)
}

func (api *genericApi) mirrorReferenceList(ctx context.Context, reference string) ([]string, error) {
var referenceList []string

referenceParts, err := api.commonApi.parseReferenceParts(reference)
if err != nil {
return nil, fmt.Errorf("unable to parse reference %q: %w", reference, err)
}

// nothing if container registry is not Docker Hub
if referenceParts.registry != name.DefaultRegistry {
return nil, nil
}

mirrors, err := api.getOrCreateRegistryMirrors(ctx)
if err != nil {
return nil, err
}

for _, mirrorRegistry := range mirrors {
mirrorRegistryUrl, err := url.Parse(mirrorRegistry)
if err != nil {
return nil, fmt.Errorf("unable to parse mirror registry url %q: %w", mirrorRegistry, err)
}

mirrorReference := mirrorRegistryUrl.Host
mirrorReference += "/" + referenceParts.repository
mirrorReference += ":" + referenceParts.tag

if referenceParts.digest != "" {
mirrorReference += "@" + referenceParts.digest
}

referenceList = append(referenceList, mirrorReference)
}

return referenceList, nil
}

func (api *genericApi) getOrCreateRegistryMirrors(ctx context.Context) ([]string, error) {
api.mutex.Lock()
defer api.mutex.Unlock()

if api.mirrors == nil {
var mirrors []string

// init registry mirrors if docker cli initialized in context
if docker.IsEnabled() && docker.IsContext(ctx) {
info, err := docker.Info(ctx)
if err != nil {
return nil, fmt.Errorf("unable to get docker system info: %w", err)
}

if info.RegistryConfig != nil {
mirrors = info.RegistryConfig.Mirrors
}
}

api.mirrors = &mirrors
}

return *api.mirrors, nil
}
10 changes: 7 additions & 3 deletions pkg/docker_registry/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
"github.com/werf/logboek"
)

var generic *api
var generic *genericApi

func Init(ctx context.Context, insecureRegistry, skipTlsVerifyRegistry bool) error {
if logboek.Context(ctx).Debug().IsAccepted() {
Expand All @@ -28,15 +28,19 @@ func Init(ctx context.Context, insecureRegistry, skipTlsVerifyRegistry bool) err
logs.Debug.SetOutput(ioutil.Discard)
}

generic = newAPI(apiOptions{
var err error
generic, err = newGenericApi(ctx, apiOptions{
InsecureRegistry: insecureRegistry,
SkipTlsVerifyRegistry: skipTlsVerifyRegistry,
})
if err != nil {
return err
}

return nil
}

func API() *api {
func API() *genericApi {
return generic
}

Expand Down

0 comments on commit bb741a3

Please sign in to comment.