Added support for Kaniko external registries (#1495)

hack/k8s/helm/nuclio/templates/_helpers.tpl

@@ -41,8 +41,12 @@
 {{- end -}}
 
 {{- define "nuclio.registryCredentialsName" -}}
+{{- if .Values.registry.secretName -}}
+{{- .Values.registry.secretName -}}
+{{- else -}}
 {{- printf "%s-registry-credentials" .Release.Name -}}
 {{- end -}}
+{{- end -}}
 
 {{- define "nuclio.registryPushPullUrlName" -}}
 {{- printf "%s-registry-url" .Release.Name -}}

hack/k8s/helm/nuclio/templates/deployment/dashboard.yaml

@@ -58,10 +58,22 @@ spec:
           value: {{ template "nuclio.dashboardName" . }}
         - name: NUCLIO_CONTAINER_BUILDER_KIND
           value: {{ .Values.dashboard.containerBuilderKind }}
-        {{- if .Values.dashboard.insecureRegistry }}
-        - name: NUCLIO_KANIKO_INSECURE_REGISTRY
+        {{- if .Values.dashboard.insecurePushRegistry }}
+        - name: NUCLIO_KANIKO_INSECURE_PUSH_REGISTRY
           value: "true"
         {{- end }}
+        {{- if .Values.dashboard.insecurePullRegistry }}
+        - name: NUCLIO_KANIKO_INSECURE_PULL_REGISTRY
+          value: "true"
+        {{- end }}
+        {{- if .Values.registry.defaultBaseRegistryURL }}
+        - name: NUCLIO_DASHBOARD_DEFAULT_BASE_REGISTRY_URL
+          value: {{ .Values.registry.defaultBaseRegistryURL }}
+        {{- end }}
+        {{- if .Values.registry.cacheRepo }}
+        - name: NUCLIO_DASHBOARD_KANIKO_CACHE_REPO
+          value: {{ .Values.registry.cacheRepo }}
+        {{- end }}
 #        - name: NUCLIO_DASHBOARD_RUN_REGISTRY_URL
 #          value: "localhost:5000"
         {{- if eq .Values.dashboard.baseImagePullPolicy "Never" }}
@@ -80,6 +92,8 @@ spec:
         - name: NUCLIO_TEMPLATES_GIT_REF
           value: "nil"
         {{- end }}
+        - name: NUCLIO_REGISTRY_CREDENTIALS_SECRET_NAME
+          value: {{ template "nuclio.registryCredentialsName" . }}
         - name: NUCLIO_DASHBOARD_EXTERNAL_IP_ADDRESSES
           value: {{ .Values.dashboard.externalIPAddresses | join "," | quote }}
         - name: NUCLIO_DASHBOARD_HTTP_INGRESS_HOST_TEMPLATE

hack/k8s/helm/nuclio/values.yaml

@@ -28,7 +28,12 @@ dashboard:
 
   # Supported container builders: "kaniko", "docker"
   containerBuilderKind: "docker"
-  insecureRegistry: false
+
+  # Set this flag to push images to a plain HTTP registry (kaniko only)
+  insecurePushRegistry: false
+
+  # Set this flag to pull images from a plain HTTP registry (kaniko only)
+  insecurePullRegistry: false
 
   # Uncomment to configure node port
   # nodePort: 32050
@@ -77,7 +82,16 @@ registry:
   # the credentials stanza below. The chart will create a secret for you (note: the credentials will be
   # visible to anyone with access to helm) and name it after the release
   #
-  secretName: registry-credentials
+  # IMPORTANT:
+  # K8S doesn't allow secret sharing between namespaces, so in order to deploy a nuclio function
+  # to namespace which differs from a namespace in which nuclio is running,
+  # secret must be copied to that namespace:
+  #
+  #  kubectl get secret <secret-name> -n <source-namespace> -o yaml \
+  #     | sed s/"namespace: <source-namespace>"/"namespace: <destination-namespace>"/ \
+  #     | kubectl apply -n <destination-namespace> -f -
+  #
+  # secretName: registry-credentials
 
   # In some cases the docker server URL in the registry secrets isn't the same as the URL with which
   # you push and pull. For example, in GKE you login to `gcr.io` (or some other regional URL) yet have
@@ -96,6 +110,14 @@ registry:
     # username: someuser
     # password: somepass
 
+  #  Use dedicated base/onbuild images registry (pull registry)
+  #
+  # defaultBaseRegistryURL: someurl
+
+  #  Set this flag to specify a remote repository that will be used to store cached layers (kaniko only)
+  #
+  # cacheRepo: someurl
+
 rbac:
 
   # If true, creates the necessary roles and role bindings for nuclio's service account

pkg/containerimagebuilderpusher/containerimagebuilder.go

@@ -16,4 +16,7 @@ type BuilderPusher interface {
 
 	// GetBaseImageRegistry returns onbuild base registry
 	GetBaseImageRegistry(registry string) string
+
+	// GetDefaultRegistryCredentialsSecretName returns secret with credentials to push/pull from docker registry
+	GetDefaultRegistryCredentialsSecretName() string
 }

pkg/containerimagebuilderpusher/docker.go

@@ -19,20 +19,22 @@ const (
 )
 
 type Docker struct {
-	dockerClient dockerclient.Client
-	logger       logger.Logger
+	dockerClient         dockerclient.Client
+	logger               logger.Logger
+	builderConfiguration *ContainerBuilderConfiguration
 }
 
-func NewDocker(logger logger.Logger) (*Docker, error) {
+func NewDocker(logger logger.Logger, builderConfiguration *ContainerBuilderConfiguration) (*Docker, error) {
 
 	dockerClient, err := dockerclient.NewShellClient(logger, nil)
 	if err != nil {
 		return nil, errors.Wrap(err, "Failed to create docker client")
 	}
 
 	dockerBuilder := &Docker{
-		dockerClient: dockerClient,
-		logger:       logger,
+		dockerClient:         dockerClient,
+		logger:               logger,
+		builderConfiguration: builderConfiguration,
 	}
 
 	return dockerBuilder, nil
@@ -71,6 +73,10 @@ func (d *Docker) GetOnbuildStages(onbuildArtifacts []runtime.Artifact) ([]string
 	return []string{}, nil
 }
 
+func (d *Docker) GetDefaultRegistryCredentialsSecretName() string {
+	return d.builderConfiguration.DefaultRegistryCredentialsSecretName
+}
+
 func (d *Docker) TransformOnbuildArtifactPaths(onbuildArtifacts []runtime.Artifact) (map[string]string, error) {
 
 	// maps between a _relative_ path in staging to the path in the image

pkg/containerimagebuilderpusher/kaniko.go

@@ -94,6 +94,10 @@ ARG NUCLIO_ARCH
 	return onbuildStages, nil
 }
 
+func (k *Kaniko) GetDefaultRegistryCredentialsSecretName() string {
+	return k.builderConfiguration.DefaultRegistryCredentialsSecretName
+}
+
 func (k *Kaniko) TransformOnbuildArtifactPaths(onbuildArtifacts []runtime.Artifact) (map[string]string, error) {
 
 	stagedArtifactPaths := make(map[string]string)
@@ -117,7 +121,7 @@ func (k *Kaniko) TransformOnbuildArtifactPaths(onbuildArtifacts []runtime.Artifa
 }
 
 func (k *Kaniko) GetBaseImageRegistry(registry string) string {
-	return registry
+	return k.builderConfiguration.DefaultBaseRegistryURL
 }
 
 func (k *Kaniko) createContainerBuildBundle(image string, contextDir string, tempDir string) (string, string, error) {
@@ -171,17 +175,23 @@ func (k *Kaniko) getKanikoJobSpec(namespace string, buildOptions *BuildOptions,
 		buildArgs = append(buildArgs, "--cache=true")
 	}
 
-	if k.builderConfiguration.InsecureRegistry {
+	if k.builderConfiguration.InsecurePushRegistry {
 		buildArgs = append(buildArgs, "--insecure")
+	}
+	if k.builderConfiguration.InsecurePullRegistry {
 		buildArgs = append(buildArgs, "--insecure-pull")
 	}
 
+	if k.builderConfiguration.CacheRepo != "" {
+		buildArgs = append(buildArgs, fmt.Sprintf("--cache-repo=%s", k.builderConfiguration.CacheRepo))
+	}
+
 	// Add build options args
 	for k, v := range buildOptions.BuildArgs {
 		buildArgs = append(buildArgs, fmt.Sprintf("--build-arg=%s=%s", k, v))
 	}
 
-	volumeMount := v1.VolumeMount{
+	tmpFolderVolumeMount := v1.VolumeMount{
 		Name:      "tmp",
 		MountPath: "/tmp",
 	}
@@ -218,7 +228,7 @@ func (k *Kaniko) getKanikoJobSpec(namespace string, buildOptions *BuildOptions,
 							Name:         "kaniko-executor",
 							Image:        k.builderConfiguration.KanikoImage,
 							Args:         buildArgs,
-							VolumeMounts: []v1.VolumeMount{volumeMount},
+							VolumeMounts: []v1.VolumeMount{tmpFolderVolumeMount},
 						},
 					},
 					InitContainers: []v1.Container{
@@ -231,7 +241,7 @@ func (k *Kaniko) getKanikoJobSpec(namespace string, buildOptions *BuildOptions,
 								"-P",
 								"/tmp",
 							},
-							VolumeMounts: []v1.VolumeMount{volumeMount},
+							VolumeMounts: []v1.VolumeMount{tmpFolderVolumeMount},
 						},
 						{
 							Name:  "extract-bundle",
@@ -243,7 +253,7 @@ func (k *Kaniko) getKanikoJobSpec(namespace string, buildOptions *BuildOptions,
 								"-C",
 								"/",
 							},
-							VolumeMounts: []v1.VolumeMount{volumeMount},
+							VolumeMounts: []v1.VolumeMount{tmpFolderVolumeMount},
 						},
 					},
 					Volumes: []v1.Volume{
@@ -260,6 +270,33 @@ func (k *Kaniko) getKanikoJobSpec(namespace string, buildOptions *BuildOptions,
 		},
 	}
 
+	// if SecretName is defined - configure mount with docker credentials
+	if !(k.builderConfiguration.InsecurePushRegistry && k.builderConfiguration.InsecurePullRegistry) &&
+		len(buildOptions.SecretName) > 0 {
+
+		kanikoJobSpec.Spec.Template.Spec.Containers[0].VolumeMounts =
+			append(kanikoJobSpec.Spec.Template.Spec.Containers[0].VolumeMounts, v1.VolumeMount{
+				Name:      "docker-config",
+				MountPath: "/kaniko/.docker",
+				ReadOnly:  true,
+			})
+
+		kanikoJobSpec.Spec.Template.Spec.Volumes = append(kanikoJobSpec.Spec.Template.Spec.Volumes, v1.Volume{
+			Name: "docker-config",
+			VolumeSource: v1.VolumeSource{
+				Secret: &v1.SecretVolumeSource{
+					SecretName: buildOptions.SecretName,
+					Items: []v1.KeyToPath{
+						{
+							Key:  ".dockerconfigjson",
+							Path: "config.json",
+						},
+					},
+				},
+			},
+		})
+	}
+
 	return kanikoJobSpec
 }
 
@@ -275,7 +312,13 @@ func (k *Kaniko) waitForKanikoJobCompletion(namespace string, jobName string, Bu
 		}
 
 		if runningJob.Status.Succeeded > 0 {
-			k.logger.Debug("Kaniko job was completed successfully")
+			jobLogs, err := k.getJobLogs(namespace, jobName)
+			if err != nil {
+				k.logger.Debug("Kaniko job was completed successfully but failed to retrieve job logs")
+				return nil
+			}
+
+			k.logger.Debug("Kaniko job was completed successfully", "jobLogs", jobLogs)
 			return nil
 		}
 		if runningJob.Status.Failed > 0 {
@@ -290,7 +333,7 @@ func (k *Kaniko) waitForKanikoJobCompletion(namespace string, jobName string, Bu
 	}
 	jobLogs, err := k.getJobLogs(namespace, jobName)
 	if err != nil {
-		return errors.Wrap(err, "Failed to retrieve kaniko job logs")
+		return errors.Wrap(err, "Kaniko job failed and was unable to retrieve job logs")
 	}
 	return fmt.Errorf("Kaniko job has timed out. Job logs:\n%s", jobLogs)
 }

pkg/containerimagebuilderpusher/types.go

@@ -12,14 +12,19 @@ type BuildOptions struct {
 	NoBaseImagePull     bool
 	BuildArgs           map[string]string
 	RegistryURL         string
+	SecretName          string
 	OutputImageFile     string
 	BuildTimeoutSeconds int64
 }
 
 type ContainerBuilderConfiguration struct {
-	Kind             string
-	BusyBoxImage     string
-	KanikoImage      string
-	JobPrefix        string
-	InsecureRegistry bool
+	Kind                                 string
+	BusyBoxImage                         string
+	KanikoImage                          string
+	JobPrefix                            string
+	DefaultRegistryCredentialsSecretName string
+	DefaultBaseRegistryURL               string
+	CacheRepo                            string
+	InsecurePushRegistry                 bool
+	InsecurePullRegistry                 bool
 }

pkg/dockercreds/dockercred.go

@@ -232,7 +232,7 @@ func extractMetaFromKeyPath(keyPath string) (string, string, string, error) {
 
 func (dc *dockerCred) login() error {
 	dc.dockerCreds.logger.DebugWith("Logging in",
-		"url", dc.path,
+		"path", dc.path,
 		"user", dc.credentials.Username)
 
 	// try to login

pkg/functionconfig/types.go

@@ -231,6 +231,7 @@ type Spec struct {
 	Alias                   string                  `json:"alias,omitempty"`
 	Build                   Build                   `json:"build,omitempty"`
 	RunRegistry             string                  `json:"runRegistry,omitempty"`
+	ImagePullSecrets        string                  `json:"imagePullSecrets,omitempty"`
 	RuntimeAttributes       map[string]interface{}  `json:"runtimeAttributes,omitempty"`
 	LoggerSinks             []LoggerSink            `json:"loggerSinks,omitempty"`
 	DealerURI               string                  `json:"dealerURI,omitempty"`

pkg/platform/abstract/platform.go

@@ -114,6 +114,10 @@ func (ap *Platform) HandleDeployFunction(existingFunctionConfig *functionconfig.
 		return nil, errors.New("Non existing function cannot be created with neverBuild mode")
 	}
 
+	if createFunctionOptions.FunctionConfig.Spec.ImagePullSecrets == "" {
+		createFunctionOptions.FunctionConfig.Spec.ImagePullSecrets = ap.platform.GetDefaultRegistryCredentialsSecretName()
+	}
+
 	// clear build mode
 	createFunctionOptions.FunctionConfig.Spec.Build.Mode = ""
 
@@ -391,6 +395,11 @@ func (ap *Platform) GetBaseImageRegistry(registry string) string {
 	return ap.ContainerBuilder.GetBaseImageRegistry(registry)
 }
 
+// // GetDefaultRegistryCredentialsSecretName returns secret with credentials to push/pull from docker registry
+func (ap *Platform) GetDefaultRegistryCredentialsSecretName() string {
+	return ap.ContainerBuilder.GetDefaultRegistryCredentialsSecretName()
+}
+
 func (ap *Platform) functionBuildRequired(createFunctionOptions *platform.CreateFunctionOptions) (bool, error) {
 
 	// if neverBuild was passed explicitly don't build

pkg/platform/factory/factory.go

@@ -136,23 +136,35 @@ func getContainerBuilderConfiguration(platformConfiguration interface{}) *contai
 
 	// if some of the parameters are undefined, try environment variables
 	if containerBuilderConfiguration.Kind == "" {
-		containerBuilderConfiguration.Kind = common.GetEnvOrDefaultString("NUCLIO_CONTAINER_BUILDER_KIND", "docker")
+		containerBuilderConfiguration.Kind = common.GetEnvOrDefaultString("NUCLIO_CONTAINER_BUILDER_KIND",
+			"docker")
 	}
 	if containerBuilderConfiguration.BusyBoxImage == "" {
-		containerBuilderConfiguration.BusyBoxImage = common.GetEnvOrDefaultString("NUCLIO_BUSYBOX_CONTAINER_IMAGE", "busybox:1.31.0")
+		containerBuilderConfiguration.BusyBoxImage = common.GetEnvOrDefaultString("NUCLIO_BUSYBOX_CONTAINER_IMAGE",
+			"busybox:1.31")
 	}
 	if containerBuilderConfiguration.KanikoImage == "" {
 		containerBuilderConfiguration.KanikoImage = common.GetEnvOrDefaultString("NUCLIO_KANIKO_CONTAINER_IMAGE",
-			"gcr.io/kaniko-project/executor:v0.13.0")
+			"gcr.io/kaniko-project/executor:v0.17.1")
 	}
 	if containerBuilderConfiguration.JobPrefix == "" {
-		containerBuilderConfiguration.JobPrefix = common.GetEnvOrDefaultString("NUCLIO_DASHBOARD_JOB_NAME_PREFIX", "kanikojob")
+		containerBuilderConfiguration.JobPrefix = common.GetEnvOrDefaultString("NUCLIO_DASHBOARD_JOB_NAME_PREFIX",
+			"kanikojob")
 	}
 
-	if common.GetEnvOrDefaultBool("NUCLIO_KANIKO_INSECURE_REGISTRY", false) {
-		containerBuilderConfiguration.InsecureRegistry = true
+	containerBuilderConfiguration.InsecurePushRegistry = common.GetEnvOrDefaultBool("NUCLIO_KANIKO_INSECURE_PUSH_REGISTRY", false)
+	containerBuilderConfiguration.InsecurePullRegistry = common.GetEnvOrDefaultBool("NUCLIO_KANIKO_INSECURE_PULL_REGISTRY", false)
+
+	containerBuilderConfiguration.DefaultRegistryCredentialsSecretName =
+		common.GetEnvOrDefaultString("NUCLIO_REGISTRY_CREDENTIALS_SECRET_NAME", "")
+
+	if containerBuilderConfiguration.DefaultBaseRegistryURL == "" {
+		containerBuilderConfiguration.DefaultBaseRegistryURL =
+			common.GetEnvOrDefaultString("NUCLIO_DASHBOARD_DEFAULT_BASE_REGISTRY_URL", "quay.io")
 	}
 
+	containerBuilderConfiguration.CacheRepo = common.GetEnvOrDefaultString("NUCLIO_DASHBOARD_KANIKO_CACHE_REPO", "")
+
 	return &containerBuilderConfiguration
 }
 

pkg/platform/kube/functionres/lazy.go

@@ -536,6 +536,10 @@ func (lc *lazyClient) createOrUpdateDeployment(functionLabels labels.Set,
 		return (resource).(*apps_v1beta1.Deployment).ObjectMeta.DeletionTimestamp != nil
 	}
 
+	if function.Spec.ImagePullSecrets != "" {
+		imagePullSecrets = function.Spec.ImagePullSecrets
+	}
+
 	createDeployment := func() (interface{}, error) {
 		method := createDeploymentResourceMethod
 		container := v1.Container{Name: "nuclio"}

pkg/platform/kube/platform.go

@@ -108,7 +108,7 @@ func NewPlatform(parentLogger logger.Logger, kubeconfigPath string,
 	} else {
 
 		// Default container image builder
-		newPlatform.ContainerBuilder, err = containerimagebuilderpusher.NewDocker(newPlatform.Logger)
+		newPlatform.ContainerBuilder, err = containerimagebuilderpusher.NewDocker(newPlatform.Logger, containerBuilderConfiguration)
 		if err != nil {
 			return nil, errors.Wrap(err, "Failed to create docker builder")
 		}