feat(kube-run): --auto-pull-secret provides private registry access for pod

Signed-off-by: Ilya Lesikov <ilya@lesikov.com>

Author: ilya-lesikov

cmd/werf/kube_run/kube_run.go

@@ -2,12 +2,17 @@ package kube_run
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"math/rand"
 	"os"
 	"os/exec"
 	"strings"
 
+	"github.com/containers/image/v5/docker/reference"
+	config2 "github.com/containers/image/v5/pkg/docker/config"
+	imgtypes "github.com/containers/image/v5/types"
 	"github.com/spf13/cobra"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -40,6 +45,7 @@ type cmdDataType struct {
 	AllocateTty     bool
 	Rm              bool
 	RmWithNamespace bool
+	AutoPullSecret  bool
 
 	Pod          string
 	Command      []string
@@ -53,6 +59,15 @@ var (
 	commonCmdData common.CmdData
 )
 
+type dockerConfigJson struct {
+	Auths map[string]dockerAuthJson `json:"auths"`
+}
+
+type dockerAuthJson struct {
+	Auth          string `json:"auth,omitempty"`
+	IdentityToken string `json:"identitytoken,omitempty"`
+}
+
 func NewCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:                   "kube-run [options] [IMAGE_NAME] [-- COMMAND ARG...]",
@@ -153,10 +168,11 @@ func NewCmd() *cobra.Command {
 	cmd.Flags().StringVarP(&cmdData.Pod, "pod", "", os.Getenv("WERF_POD"), "Set created pod name (default $WERF_POD or autogenerated if not specified)")
 	cmd.Flags().StringVarP(&cmdData.Overrides, "overrides", "", os.Getenv("WERF_OVERRIDES"), "Inline JSON to override/extend any fields in created Pod, e.g. to add imagePullSecrets field (default $WERF_OVERRIDES)")
 	cmd.Flags().StringVarP(&cmdData.ExtraOptions, "extra-options", "", os.Getenv("WERF_EXTRA_OPTIONS"), "Pass extra options to \"kubectl run\" command (default $WERF_EXTRA_OPTIONS)")
-	cmd.Flags().BoolVarP(&cmdData.Rm, "rm", "", common.GetBoolEnvironmentDefaultTrue("WERF_RM"), "Remove pod after command completion (default $WERF_RM or true if not specified)")
+	cmd.Flags().BoolVarP(&cmdData.Rm, "rm", "", common.GetBoolEnvironmentDefaultTrue("WERF_RM"), "Remove pod and other created resources after command completion (default $WERF_RM or true if not specified)")
 	cmd.Flags().BoolVarP(&cmdData.RmWithNamespace, "rm-with-namespace", "", common.GetBoolEnvironmentDefaultFalse("WERF_RM_WITH_NAMESPACE"), "Remove also a namespace after command completion (default $WERF_RM_WITH_NAMESPACE or false if not specified)")
 	cmd.Flags().BoolVarP(&cmdData.Interactive, "interactive", "i", common.GetBoolEnvironmentDefaultFalse("WERF_INTERACTIVE"), "Enable interactive mode (default $WERF_INTERACTIVE or false if not specified)")
 	cmd.Flags().BoolVarP(&cmdData.AllocateTty, "tty", "t", common.GetBoolEnvironmentDefaultFalse("WERF_TTY"), "Allocate a TTY (default $WERF_TTY or false if not specified)")
+	cmd.Flags().BoolVarP(&cmdData.AutoPullSecret, "auto-pull-secret", "", common.GetBoolEnvironmentDefaultTrue("WERF_AUTO_PULL_SECRET"), "Automatically create docker config secret in the namespace and plug it via pod's imagePullSecrets for private registry access (default $WERF_AUTO_PULL_SECRET or true if not specified)")
 
 	return cmd
 }
@@ -252,6 +268,7 @@ func runMain(ctx context.Context) error {
 	} else {
 		pod = cmdData.Pod
 	}
+	secret := pod
 
 	_, werfConfig, err := common.GetRequiredWerfConfig(ctx, &commonCmdData, giterminismManager, common.GetWerfConfigOptions(&commonCmdData, false))
 	if err != nil {
@@ -271,31 +288,31 @@ func runMain(ctx context.Context) error {
 	}
 
 	defer func() {
-		cleanupResources(ctx, pod, namespace)
+		cleanupResources(ctx, pod, secret, namespace)
 	}()
 
 	if *commonCmdData.Follow {
 		return common.FollowGitHead(ctx, &commonCmdData, func(ctx context.Context, headCommitGiterminismManager giterminism_manager.Interface) error {
-			cleanupResources(ctx, pod, namespace)
+			cleanupResources(ctx, pod, secret, namespace)
 
-			if err := run(ctx, pod, namespace, werfConfig, containerRuntime, giterminismManager); err != nil {
+			if err := run(ctx, pod, secret, namespace, werfConfig, containerRuntime, giterminismManager); err != nil {
 				return err
 			}
 
-			cleanupResources(ctx, pod, namespace)
+			cleanupResources(ctx, pod, secret, namespace)
 
 			return nil
 		})
 	} else {
-		if err := run(ctx, pod, namespace, werfConfig, containerRuntime, giterminismManager); err != nil {
+		if err := run(ctx, pod, secret, namespace, werfConfig, containerRuntime, giterminismManager); err != nil {
 			return err
 		}
 	}
 
 	return nil
 }
 
-func run(ctx context.Context, pod string, namespace string, werfConfig *config.WerfConfig, containerRuntime container_runtime.ContainerRuntime, giterminismManager giterminism_manager.Interface) error {
+func run(ctx context.Context, pod, secret, namespace string, werfConfig *config.WerfConfig, containerRuntime container_runtime.ContainerRuntime, giterminismManager giterminism_manager.Interface) error {
 	projectName := werfConfig.Meta.Project
 
 	projectTmpDir, err := tmp_manager.CreateProjectDir(ctx)
@@ -371,6 +388,19 @@ func run(ctx context.Context, pod string, namespace string, werfConfig *config.W
 		return err
 	}
 
+	if cmdData.AutoPullSecret {
+		namedRef, dockerAuthConf, err := getDockerConfigCredentials(image)
+		if err != nil {
+			return fmt.Errorf("unable to get docker config credentials: %w", err)
+		}
+
+		if dockerAuthConf != (imgtypes.DockerAuthConfig{}) {
+			if err := createDockerRegistrySecret(ctx, secret, namespace, *namedRef, dockerAuthConf); err != nil {
+				return fmt.Errorf("unable to create docker registry secret: %w", err)
+			}
+		}
+	}
+
 	args := []string{
 		"kubectl",
 		"run",
@@ -391,8 +421,26 @@ func run(ctx context.Context, pod string, namespace string, werfConfig *config.W
 		args = append(args, "-t")
 	}
 
+	var overrides map[string]interface{}
 	if cmdData.Overrides != "" {
-		args = append(args, "--overrides", cmdData.Overrides)
+		if err := json.Unmarshal([]byte(cmdData.Overrides), &overrides); err != nil {
+			return fmt.Errorf("unable to unmarshal --overrides: %w", err)
+		}
+	}
+
+	if cmdData.AutoPullSecret {
+		overrides, err = addImagePullSecret(secret, overrides)
+		if err != nil {
+			return fmt.Errorf("unable to add imagePullSecret to --overrides: %w", err)
+		}
+	}
+
+	if len(overrides) > 0 {
+		overridesB, err := json.Marshal(overrides)
+		if err != nil {
+			return fmt.Errorf("unable to marshal generated --overrides: %w", err)
+		}
+		args = append(args, "--overrides", string(overridesB))
 	}
 
 	if cmdData.ExtraOptions != "" {
@@ -431,21 +479,34 @@ func run(ctx context.Context, pod string, namespace string, werfConfig *config.W
 	})
 }
 
-func cleanupResources(ctx context.Context, pod string, namespace string) {
+func cleanupResources(ctx context.Context, pod, secret, namespace string) {
+	if !cmdData.Rm {
+		return
+	}
+
 	if isNsExist, err := isNamespaceExist(ctx, namespace); err != nil {
 		logboek.Context(ctx).Warn().LogF("WARNING: unable to check for namespace existence: %s\n", err)
 		return
 	} else if !isNsExist {
 		return
 	}
 
-	if cmdData.Rm {
-		if isPodExist, err := isPodExist(ctx, pod, namespace); err != nil {
-			logboek.Context(ctx).Warn().LogF("WARNING: unable to check for pod existence: %s\n", err)
-		} else if isPodExist {
-			logboek.Context(ctx).LogF("Cleaning up pod %q ...\n", pod)
-			if err := kube.Client.CoreV1().Pods(namespace).Delete(ctx, pod, v1.DeleteOptions{}); err != nil {
-				logboek.Context(ctx).Warn().LogF("WARNING: pod cleaning up failed: %s\n", err)
+	if isPodExist, err := isPodExist(ctx, pod, namespace); err != nil {
+		logboek.Context(ctx).Warn().LogF("WARNING: unable to check for pod existence: %s\n", err)
+	} else if isPodExist {
+		logboek.Context(ctx).LogF("Cleaning up pod %q ...\n", pod)
+		if err := kube.Client.CoreV1().Pods(namespace).Delete(ctx, pod, v1.DeleteOptions{}); err != nil {
+			logboek.Context(ctx).Warn().LogF("WARNING: pod cleaning up failed: %s\n", err)
+		}
+	}
+
+	if cmdData.AutoPullSecret {
+		if isSecretExist, err := isSecretExist(ctx, secret, namespace); err != nil {
+			logboek.Context(ctx).Warn().LogF("WARNING: unable to check for secret existence: %s\n", err)
+		} else if isSecretExist {
+			logboek.Context(ctx).LogF("Cleaning up secret %q ...\n", secret)
+			if err := kube.Client.CoreV1().Secrets(namespace).Delete(ctx, secret, v1.DeleteOptions{}); err != nil {
+				logboek.Context(ctx).Warn().LogF("WARNING: secret cleaning up failed: %s\n", err)
 			}
 		}
 	}
@@ -480,6 +541,45 @@ func createNamespace(ctx context.Context, namespace string) error {
 	return nil
 }
 
+func createDockerRegistrySecret(ctx context.Context, name, namespace string, ref reference.Named, dockerAuthConf imgtypes.DockerAuthConfig) error {
+	secret := &corev1.Secret{
+		ObjectMeta: v1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Data: map[string][]byte{},
+		Type: corev1.SecretTypeDockerConfigJson,
+	}
+
+	var authJson dockerAuthJson
+	switch {
+	case dockerAuthConf.IdentityToken != "":
+		authJson.IdentityToken = dockerAuthConf.IdentityToken
+	case dockerAuthConf.Username != "" && dockerAuthConf.Password != "":
+		authJson.Auth = base64.StdEncoding.EncodeToString([]byte(dockerAuthConf.Username + ":" + dockerAuthConf.Password))
+	default:
+		panic("unexpected dockerAuthConf")
+	}
+
+	dockerConfJson := &dockerConfigJson{
+		Auths: map[string]dockerAuthJson{
+			ref.Name(): authJson,
+		},
+	}
+
+	dockerConf, err := json.Marshal(dockerConfJson)
+	if err != nil {
+		return fmt.Errorf("unable to marshal docker config json: %w", err)
+	}
+
+	secret.Data[corev1.DockerConfigJsonKey] = dockerConf
+
+	logboek.Context(ctx).LogF("Creating secret %q in namespace %q ...\n", name, namespace)
+	kube.Client.CoreV1().Secrets(namespace).Create(ctx, secret, v1.CreateOptions{})
+
+	return nil
+}
+
 func isNamespaceExist(ctx context.Context, namespace string) (bool, error) {
 	if matchedNamespaces, err := kube.Client.CoreV1().Namespaces().List(ctx, v1.ListOptions{
 		FieldSelector: fields.OneTermEqualSelector("metadata.name", namespace).String(),
@@ -503,3 +603,74 @@ func isPodExist(ctx context.Context, pod string, namespace string) (bool, error)
 
 	return false, nil
 }
+
+func isSecretExist(ctx context.Context, secret string, namespace string) (bool, error) {
+	if matchedSecrets, err := kube.Client.CoreV1().Secrets(namespace).List(ctx, v1.ListOptions{
+		FieldSelector: fields.OneTermEqualSelector("metadata.name", secret).String(),
+	}); err != nil {
+		return false, fmt.Errorf("unable to list secrets: %w", err)
+	} else if len(matchedSecrets.Items) > 0 {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+// Might return empty DockerAuthConfig.
+func getDockerConfigCredentials(ref string) (*reference.Named, imgtypes.DockerAuthConfig, error) {
+	namedRef, err := reference.ParseNormalizedNamed(ref)
+	if err != nil {
+		return nil, imgtypes.DockerAuthConfig{}, fmt.Errorf("unable to parse docker config registry reference %q: %w", ref, err)
+	}
+
+	dockerAuthConf, err := config2.GetCredentialsForRef(&imgtypes.SystemContext{AuthFilePath: *commonCmdData.DockerConfig}, namedRef)
+	if err != nil {
+		return nil, imgtypes.DockerAuthConfig{}, fmt.Errorf("unable to get docker registry creds for ref %q: %w", ref, err)
+	}
+
+	return &namedRef, dockerAuthConf, nil
+}
+
+func addImagePullSecret(secret string, overrides map[string]interface{}) (map[string]interface{}, error) {
+	if secret == "" {
+		panic("secret name can't be empty")
+	}
+
+	newImagePullSecret := map[string]interface{}{"name": secret}
+	newImagePullSecrets := []interface{}{newImagePullSecret}
+	newSpec := map[string]interface{}{"imagePullSecrets": newImagePullSecrets}
+	newOverrides := map[string]interface{}{"spec": newSpec}
+
+	if len(overrides) == 0 {
+		return newOverrides, nil
+	}
+
+	if _, ok := overrides["spec"]; !ok {
+		overrides["spec"] = newSpec
+		return overrides, nil
+	}
+
+	overridesSpec, ok := overrides["spec"].(map[string]interface{})
+	if !ok {
+		return nil, fmt.Errorf("unexpected pod spec overrides format: %+v", overrides)
+	}
+
+	if len(overridesSpec) == 0 {
+		overrides["spec"] = newSpec
+		return overrides, nil
+	}
+
+	_, ok = overridesSpec["imagePullSecrets"]
+	if !ok {
+		overrides["spec"].(map[string]interface{})["imagePullSecrets"] = newImagePullSecrets
+		return overrides, nil
+	}
+
+	_, ok = overridesSpec["imagePullSecrets"].([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("unexpected imagePullSecrets overrides format: %+v", overrides)
+	}
+
+	overrides["spec"].(map[string]interface{})["imagePullSecrets"] = append(overrides["spec"].(map[string]interface{})["imagePullSecrets"].([]interface{}), newImagePullSecret)
+	return overrides, nil
+}