From 1bf529e6d9d275f57581bb51fa3b476cdc636eda Mon Sep 17 00:00:00 2001 From: Damien Degois Date: Thu, 15 Feb 2024 00:23:41 +0100 Subject: [PATCH] Add flag to remap registries for any registry mirror (#2935) * Add flag to remap registries for any registry mirror The purpose of this PR is to add an option to remap registries, a kind of generalized `--registry-mirror`. This is helpful for air-gapped environments and/or when local registry mirrors are available (not limited to docker.io). This allows user to reference any images without having to change their location. It also permit to separate infra related configuration (the mirrors) from CI/CD pipeline definition by using an environment variable for example (the reason behind the early return if flag provided but empty). Therefore you can have a pipeline calling kaniko with `--registry-map=$REGISTRY_MAP` and have the `REGISTRY_MAP` populated via the runner's env by another team, and the absence of env wouldn't trigger a failure, it makes the pipeline env independent. I've also considered the option of environment variables directly but it doesn't seems to be in kaniko's philosophy. This makes quite some duplicated code :/ One option to keep the mirror flag and behavior would be to use only one codebase and convert `--registry-mirror=VALUE` to `--registry-map=index.docker.io=VALUE` internally. Suggestions welcome! * Configure logging config sooner to be able to use it in flag parsing * Replace registry mirrors by maps logic and use env var * Add env vars to README.md * Fix test --- README.md | 30 ++++++++++++++++++++--- cmd/executor/cmd/root.go | 26 +++++++++++++++++--- integration/integration_test.go | 40 ++++++++++++++++++++++++++++++ pkg/config/args.go | 43 +++++++++++++++++++++++++++++++++ pkg/config/args_test.go | 33 +++++++++++++++++++++++++ pkg/config/options.go | 1 + pkg/image/remote/remote.go | 23 +++++++++--------- pkg/image/remote/remote_test.go | 2 +- 8 files changed, 178 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 822936ed51..2601139fe9 100644 --- a/README.md +++ b/README.md @@ -95,12 +95,13 @@ _If you are interested in contributing to kaniko, see - [Flag `--push-retry`](#flag---push-retry) - [Flag `--registry-certificate`](#flag---registry-certificate) - [Flag `--registry-client-cert`](#flag---registry-client-cert) + - [Flag `--registry-map`](#flag---registry-map) - [Flag `--registry-mirror`](#flag---registry-mirror) - [Flag `--skip-default-registry-fallback`](#flag---skip-default-registry-fallback) - [Flag `--reproducible`](#flag---reproducible) - [Flag `--single-snapshot`](#flag---single-snapshot) - - [Flag `--skip-tls-verify`](#flag---skip-tls-verify) - [Flag `--skip-push-permission-check`](#flag---skip-push-permission-check) + - [Flag `--skip-tls-verify`](#flag---skip-tls-verify) - [Flag `--skip-tls-verify-pull`](#flag---skip-tls-verify-pull) - [Flag `--skip-tls-verify-registry`](#flag---skip-tls-verify-registry) - [Flag `--skip-unused-stages`](#flag---skip-unused-stages) @@ -112,8 +113,8 @@ _If you are interested in contributing to kaniko, see - [Flag `--ignore-var-run`](#flag---ignore-var-run) - [Flag `--ignore-path`](#flag---ignore-path) - [Flag `--image-fs-extract-retry`](#flag---image-fs-extract-retry) - - [Flag `--image-download-retry`](#flag---image-download-retry) - - [Debug Image](#debug-image) + - [Flag `--image-download-retry`](#flag---image-download-retry) + - [Debug Image](#debug-image) - [Security](#security) - [Verifying Signed Kaniko Images](#verifying-signed-kaniko-images) - [Kaniko Builds - Profiling](#kaniko-builds---profiling) @@ -981,6 +982,27 @@ for authentication. Expected format is `my.registry.url=/path/to/client/cert.crt,/path/to/client/key.key` +#### Flag `--registry-map` + +Set this flag if you want to remap registries references. Usefull for air gap environement for example. +You can use this flag more than once, if you want to set multiple mirrors for a given registry. +You can mention several remap in a single flag too, separated by semi-colon. +If an image is not found on the first mirror, Kaniko will try +the next mirror(s), and at the end fallback on the original registry. + +Registry maps can also be defined through `KANIKO_REGISTRY_MAP` environment variable. + +Expected format is `original-registry=remapped-registry[;another-reg=another-remap[;...]]` for example. + +Note that you can't specify a URL with scheme for this flag. Some valid options +are: + +- `index.docker.io=mirror.gcr.io` +- `gcr.io=127.0.0.1` +- `quay.io=192.168.0.1:5000` +- `index.docker.io=docker-io.mirrors.corp.net;index.docker.io=mirror.gcr.io;gcr.io=127.0.0.1` + will try `docker-io.mirrors.corp.net` then `mirror.gcr.io` for `index.docker.io` and `127.0.0.1` for `gcr.io` + #### Flag `--registry-mirror` Set this flag if you want to use a registry mirror instead of the default @@ -988,6 +1010,8 @@ Set this flag if you want to use a registry mirror instead of the default multiple mirrors. If an image is not found on the first mirror, Kaniko will try the next mirror(s), and at the end fallback on the default registry. +Mirror can also be defined through `KANIKO_REGISTRY_MIRROR` environment variable. + Expected format is `mirror.gcr.io` for example. Note that you can't specify a URL with scheme for this flag. Some valid options diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 23233ed06f..2bdc05c068 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -34,6 +34,7 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/GoogleContainerTools/kaniko/pkg/util/proc" "github.com/containerd/containerd/platforms" + "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -70,6 +71,21 @@ func validateFlags() { opts.RegistryMirrors.Set(val) } + // Allow setting --registry-maps using an environment variable. + if val, ok := os.LookupEnv("KANIKO_REGISTRY_MAP"); ok { + opts.RegistryMaps.Set(val) + } + + for _, target := range opts.RegistryMirrors { + opts.RegistryMaps.Set(fmt.Sprintf("%s=%s", name.DefaultRegistry, target)) + } + + if len(opts.RegistryMaps) > 0 { + for src, dsts := range opts.RegistryMaps { + logrus.Debugf("registry-map remaps %s to %s.", src, strings.Join(dsts, ", ")) + } + } + // Default the custom platform flag to our current platform, and validate it. if opts.CustomPlatform == "" { opts.CustomPlatform = platforms.Format(platforms.Normalize(platforms.DefaultSpec())) @@ -85,6 +101,10 @@ var RootCmd = &cobra.Command{ PersistentPreRunE: func(cmd *cobra.Command, args []string) error { if cmd.Use == "executor" { + if err := logging.Configure(logLevel, logFormat, logTimestamp); err != nil { + return err + } + validateFlags() // Command line flag takes precedence over the KANIKO_DIR environment variable. @@ -99,10 +119,6 @@ var RootCmd = &cobra.Command{ resolveEnvironmentBuildArgs(opts.BuildArgs, os.Getenv) - if err := logging.Configure(logLevel, logFormat, logTimestamp); err != nil { - return err - } - if !opts.NoPush && len(opts.Destinations) == 0 { return errors.New("you must provide --destination, or use --no-push") } @@ -238,6 +254,8 @@ func addKanikoOptionsFlags() { RootCmd.PersistentFlags().VarP(&opts.RegistriesCertificates, "registry-certificate", "", "Use the provided certificate for TLS communication with the given registry. Expected format is 'my.registry.url=/path/to/the/server/certificate'.") opts.RegistriesClientCertificates = make(map[string]string) RootCmd.PersistentFlags().VarP(&opts.RegistriesClientCertificates, "registry-client-cert", "", "Use the provided client certificate for mutual TLS (mTLS) communication with the given registry. Expected format is 'my.registry.url=/path/to/client/cert,/path/to/client/key'.") + opts.RegistryMaps = make(map[string][]string) + RootCmd.PersistentFlags().VarP(&opts.RegistryMaps, "registry-map", "", "Registry map of mirror to use as pull-through cache instead. Expected format is 'orignal.registry=new.registry;other-original.registry=other-remap.registry'") RootCmd.PersistentFlags().VarP(&opts.RegistryMirrors, "registry-mirror", "", "Registry mirror to use as pull-through cache instead of docker.io. Set it repeatedly for multiple mirrors.") RootCmd.PersistentFlags().BoolVarP(&opts.SkipDefaultRegistryFallback, "skip-default-registry-fallback", "", false, "If an image is not found on any mirrors (defined with registry-mirror) do not fallback to the default registry. If registry-mirror is not defined, this flag is ignored.") RootCmd.PersistentFlags().BoolVarP(&opts.IgnoreVarRun, "ignore-var-run", "", true, "Ignore /var/run directory when taking image snapshot. Set it to false to preserve /var/run/ in destination image.") diff --git a/integration/integration_test.go b/integration/integration_test.go index fe3adbdf6e..c0a0a6ee0a 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -396,6 +396,46 @@ func TestBuildViaRegistryMirrors(t *testing.T) { checkContainerDiffOutput(t, diff, expected) } +func TestBuildViaRegistryMap(t *testing.T) { + repo := getGitRepo(false) + dockerfile := fmt.Sprintf("%s/%s/Dockerfile_registry_mirror", integrationPath, dockerfilesPath) + + // Build with docker + dockerImage := GetDockerImage(config.imageRepo, "Dockerfile_registry_mirror") + dockerCmd := exec.Command("docker", + append([]string{"build", + "-t", dockerImage, + "-f", dockerfile, + repo})...) + out, err := RunCommandWithoutTest(dockerCmd) + if err != nil { + t.Errorf("Failed to build image %s with docker command %q: %s %s", dockerImage, dockerCmd.Args, err, string(out)) + } + + // Build with kaniko + kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_registry_mirror") + dockerRunFlags := []string{"run", "--net=host"} + dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount) + dockerRunFlags = append(dockerRunFlags, ExecutorImage, + "-f", dockerfile, + "-d", kanikoImage, + "--registry-map", "index.docker.io=doesnotexist.example.com", + "--registry-map", "index.docker.io=us-mirror.gcr.io", + "-c", fmt.Sprintf("git://%s", repo)) + + kanikoCmd := exec.Command("docker", dockerRunFlags...) + + out, err = RunCommandWithoutTest(kanikoCmd) + if err != nil { + t.Errorf("Failed to build image %s with kaniko command %q: %v %s", dockerImage, kanikoCmd.Args, err, string(out)) + } + + diff := containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--no-cache") + + expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage, dockerImage, kanikoImage) + checkContainerDiffOutput(t, diff, expected) +} + func TestBuildSkipFallback(t *testing.T) { repo := getGitRepo(false) dockerfile := fmt.Sprintf("%s/%s/Dockerfile_registry_mirror", integrationPath, dockerfilesPath) diff --git a/pkg/config/args.go b/pkg/config/args.go index 7bf52494eb..69760cbedf 100644 --- a/pkg/config/args.go +++ b/pkg/config/args.go @@ -82,3 +82,46 @@ func (a *keyValueArg) Set(value string) error { func (a *keyValueArg) Type() string { return "key-value-arg type" } + +type multiKeyMultiValueArg map[string][]string + +func (c *multiKeyMultiValueArg) parseKV(value string) error { + valueSplit := strings.SplitN(value, "=", 2) + if len(valueSplit) < 2 { + return fmt.Errorf("invalid argument value. expect key=value, got %s", value) + } + (*c)[valueSplit[0]] = append((*c)[valueSplit[0]], valueSplit[1]) + return nil +} + +func (c *multiKeyMultiValueArg) String() string { + var result []string + for key := range *c { + for _, val := range (*c)[key] { + result = append(result, fmt.Sprintf("%s=%s", key, val)) + } + } + return strings.Join(result, ";") + +} + +func (c *multiKeyMultiValueArg) Set(value string) error { + if value == "" { + return nil + } + if strings.Contains(value, ";") { + kvpairs := strings.Split(value, ";") + for _, kv := range kvpairs { + err := c.parseKV(kv) + if err != nil { + return err + } + } + return nil + } + return c.parseKV(value) +} + +func (c *multiKeyMultiValueArg) Type() string { + return "key-multi-value-arg type" +} diff --git a/pkg/config/args_test.go b/pkg/config/args_test.go index 12cf839770..4e16a55696 100644 --- a/pkg/config/args_test.go +++ b/pkg/config/args_test.go @@ -45,3 +45,36 @@ func Test_KeyValueArg_Set_shouldAcceptEqualAsValue(t *testing.T) { t.Error("Invalid split. key=value=something should be split to key=>value=something") } } + +func Test_multiKeyMultiValueArg_Set_shouldSplitArgumentLikeKVA(t *testing.T) { + arg := make(multiKeyMultiValueArg) + arg.Set("key=value") + if arg["key"][0] != "value" { + t.Error("Invalid split. key=value should be split to key=>value") + } +} + +func Test_multiKeyMultiValueArg_Set_ShouldAppendIfRepeated(t *testing.T) { + arg := make(multiKeyMultiValueArg) + arg.Set("key=v1") + arg.Set("key=v2") + if arg["key"][0] != "v1" || arg["key"][1] != "v2" { + t.Error("Invalid repeat behavior. Repeated keys should append values") + } +} + +func Test_multiKeyMultiValueArg_Set_Composed(t *testing.T) { + arg := make(multiKeyMultiValueArg) + arg.Set("key1=value1;key2=value2") + if arg["key1"][0] != "value1" || arg["key2"][0] != "value2" { + t.Error("Invalid composed value parsing. key=value;key2=value2 should generate 2 keys") + } +} + +func Test_multiKeyMultiValueArg_Set_WithEmptyValueShouldWork(t *testing.T) { + arg := make(multiKeyMultiValueArg) + err := arg.Set("") + if len(arg) != 0 || err != nil { + t.Error("multiKeyMultiValueArg must handle empty value") + } +} diff --git a/pkg/config/options.go b/pkg/config/options.go index df419ff64c..2ec4afc436 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -32,6 +32,7 @@ type CacheOptions struct { // RegistryOptions are all the options related to the registries, set by command line arguments. type RegistryOptions struct { + RegistryMaps multiKeyMultiValueArg RegistryMirrors multiArg InsecureRegistries multiArg SkipTLSVerifyRegistries multiArg diff --git a/pkg/image/remote/remote.go b/pkg/image/remote/remote.go index 2374935c7b..6fa096b9fd 100644 --- a/pkg/image/remote/remote.go +++ b/pkg/image/remote/remote.go @@ -17,7 +17,7 @@ limitations under the License. package remote import ( - "errors" + "fmt" "strings" "github.com/GoogleContainerTools/kaniko/pkg/config" @@ -51,33 +51,32 @@ func RetrieveRemoteImage(image string, opts config.RegistryOptions, customPlatfo return nil, err } - if ref.Context().RegistryStr() == name.DefaultRegistry { + if newRegURLs, found := opts.RegistryMaps[ref.Context().RegistryStr()]; found { ref, err := normalizeReference(ref, image) if err != nil { return nil, err } - for _, registryMirror := range opts.RegistryMirrors { + for _, regToMapTo := range newRegURLs { var newReg name.Registry - if opts.InsecurePull || opts.InsecureRegistries.Contains(registryMirror) { - newReg, err = name.NewRegistry(registryMirror, name.WeakValidation, name.Insecure) + if opts.InsecurePull || opts.InsecureRegistries.Contains(regToMapTo) { + newReg, err = name.NewRegistry(regToMapTo, name.WeakValidation, name.Insecure) } else { - newReg, err = name.NewRegistry(registryMirror, name.StrictValidation) + newReg, err = name.NewRegistry(regToMapTo, name.StrictValidation) } if err != nil { return nil, err } ref := setNewRegistry(ref, newReg) - - logrus.Infof("Retrieving image %s from registry mirror %s", ref, registryMirror) + logrus.Infof("Retrieving image %s from mapped registry %s", ref, regToMapTo) retryFunc := func() (v1.Image, error) { - return remoteImageFunc(ref, remoteOptions(registryMirror, opts, customPlatform)...) + return remoteImageFunc(ref, remoteOptions(regToMapTo, opts, customPlatform)...) } var remoteImage v1.Image var err error if remoteImage, err = util.RetryWithResult(retryFunc, opts.ImageDownloadRetry, 1000); err != nil { - logrus.Warnf("Failed to retrieve image %s from registry mirror %s: %s. Will try with the next mirror, or fallback to the default registry.", ref, registryMirror, err) + logrus.Warnf("Failed to retrieve image %s from remapped registry %s: %s. Will try with the next registry, or fallback to the original registry.", ref, regToMapTo, err) continue } @@ -86,8 +85,8 @@ func RetrieveRemoteImage(image string, opts config.RegistryOptions, customPlatfo return remoteImage, nil } - if len(opts.RegistryMirrors) > 0 && opts.SkipDefaultRegistryFallback { - return nil, errors.New("image not found on any configured mirror(s)") + if len(newRegURLs) > 0 && opts.SkipDefaultRegistryFallback { + return nil, fmt.Errorf("image not found on any configured mapped registries for %s", ref) } } diff --git a/pkg/image/remote/remote_test.go b/pkg/image/remote/remote_test.go index 7fb6089d4a..e68c1fe524 100644 --- a/pkg/image/remote/remote_test.go +++ b/pkg/image/remote/remote_test.go @@ -117,7 +117,7 @@ func Test_RetrieveRemoteImage_skipFallback(t *testing.T) { registryMirror := "some-registry" opts := config.RegistryOptions{ - RegistryMirrors: []string{registryMirror}, + RegistryMaps: map[string][]string{name.DefaultRegistry: {registryMirror}}, SkipDefaultRegistryFallback: false, }