Function Image Builder (#621)

* Image builder

* Docker registry library

* Documentation

* Integrate builder and registry

* Other fixes. Fix golint and gofmt. Image ref update.

* Rewrite image-builder in Go

* Fix custom-get-python test

* Enable function building with a configuration field

* Generate builder image in Travis

* Give RBAC permissions to delete jobs

* Minor refactor

* Minor review

* Minor review

* Add permission to read registry secret

* Fix binary name

* Add debug to get-python-deps-update example

.gitignore

@@ -147,6 +147,7 @@ bats/
 bundles/
 docker/controller-manager/kubeless-controller-manager
 docker/kafka-controller/kafka-controller
+docker/function-image-builder/imbuilder
 ksonnet-lib/
 kubeless-openshift.yaml
 kubeless-rbac.yaml

.travis.yml

@@ -21,8 +21,10 @@ env:
     - TEST_TARGET=GKE
   global:
     - CONTROLLER_IMAGE_NAME=bitnami/kubeless-controller-manager
+    - BUILDER_IMAGE_NAME=kubeless/function-image-builder
     - CONTROLLER_TAG=${TRAVIS_TAG:-build-$TRAVIS_BUILD_ID}
     - CONTROLLER_IMAGE=${CONTROLLER_IMAGE_NAME}:${CONTROLLER_TAG}
+    - BUILDER_IMAGE=${BUILDER_IMAGE_NAME}:${CONTROLLER_TAG}
     - KAFKA_CONTROLLER_IMAGE_NAME=bitnami/kafka-trigger-controller
     - KAFKA_CONTROLLER_IMAGE=${KAFKA_CONTROLLER_IMAGE_NAME}:${CONTROLLER_TAG}
     - CGO_ENABLED=0
@@ -94,6 +96,7 @@ script:
     if [[ "$SHOULD_TEST" == "1" ]]; then
       make VERSION=${TRAVIS_TAG:-build-$TRAVIS_BUILD_ID} binary
       make controller-image CONTROLLER_IMAGE=$CONTROLLER_IMAGE
+      make function-image-builder FUNCTION_IMAGE_BUILDER=$BUILDER_IMAGE
       make kafka-controller-image KAFKA_CONTROLLER_IMAGE=$KAFKA_CONTROLLER_IMAGE
       make all-yaml
       sed -i.bak 's/'":latest"'/'":${CONTROLLER_TAG}"'/g' kubeless.yaml
@@ -116,6 +119,7 @@ script:
       GKE)
         docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
         docker push $CONTROLLER_IMAGE
+        docker push $BUILDER_IMAGE
         docker push $KAFKA_CONTROLLER_IMAGE
         echo "Waiting for the GKE cluster to be ready"
         tail -f $TRAVIS_BUILD_DIR/gke-start.log &

Makefile

@@ -6,6 +6,7 @@ VERSION = dev-$(shell date +%FT%T%z)
 KUBECFG = kubecfg
 DOCKER = docker
 CONTROLLER_IMAGE = kubeless-controller-manager:latest
+FUNCTION_IMAGE_BUILDER = kubeless-function-image-builder:latest
 KAFKA_CONTROLLER_IMAGE = kafka-trigger-controller:latest
 OS = linux
 ARCH = amd64
@@ -57,6 +58,15 @@ controller-build:
 controller-image: docker/controller-manager
 	$(DOCKER) build -t $(CONTROLLER_IMAGE) $<
 
+docker/function-image-builder: function-image-builder-build
+	cp $(BUNDLES)/kubeless_$(OS)-$(ARCH)/imbuilder $@
+
+function-image-builder-build:
+	./script/binary-controller -os=$(OS) -arch=$(ARCH) imbuilder github.com/kubeless/kubeless/pkg/function-image-builder
+
+function-image-builder: docker/function-image-builder
+	$(DOCKER) build -t $(FUNCTION_IMAGE_BUILDER) $<
+
 docker/kafka-controller: kafka-controller-build
 	cp $(BUNDLES)/kubeless_$(OS)-$(ARCH)/kafka-controller $@
 

docker/controller-manager/Dockerfile

@@ -1,5 +1,7 @@
 FROM bitnami/minideb:jessie
 
+RUN install_packages ca-certificates
+
 ADD kubeless-controller-manager /kubeless-controller-manager
 
 ENTRYPOINT ["/kubeless-controller-manager"]

docker/function-image-builder/Dockerfile

@@ -0,0 +1,8 @@
+FROM fedora:27
+
+RUN dnf install -y skopeo nodejs
+
+ADD imbuilder /
+ADD entrypoint.sh /
+
+ENTRYPOINT [ "/entrypoint.sh" ]

docker/function-image-builder/entrypoint.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Copyright (c) 2016-2017 Bitnami
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# Kubernetes ImagePullSecrets uses .dockerconfigjson as the file name
+# for storing credentials but skopeo requires it to be named config.json
+if [ -f $DOCKER_CONFIG_FOLDER/.dockerconfigjson ]; then
+    echo "Creating $HOME/.docker/config.json"
+    mkdir -p $HOME/.docker
+    ln -s $DOCKER_CONFIG_FOLDER/.dockerconfigjson $HOME/.docker/config.json
+fi
+
+"${@}"

docs/building-functions.md

@@ -0,0 +1,64 @@
+# Kubeless building process for functions
+
+> **Warning**: This feature is still under heavy development
+
+Kubeless includes a way of building and storing functions as docker images. This can be used to:
+
+ - Persistent function storage.
+ - Speed the process of redeploying the same function. This is specicially useful for scalling your function.
+ - Generate immutable function deployments. Once a function image is generated, the same image will be used every time the function is used.
+
+## Setup the build process
+
+In order to setup the build process the only steps needed are:
+
+ 1. Generate a Kubernetes [secret](https://kubernetes.io/docs/concepts/configuration/secret) with the credentials required to push images to the docker registry and enable the build st. In order to do so, `kubectl` has an utility that allows you to create this secret in just one command:
+
+| **Note**: The command below will generate the correct secret only if the version of `kubectl` is 1.9+ 
+
+```console
+kubectl create secret docker-registry kubeless-registry-credentials \
+  --docker-server=https://index.docker.io/v1/ \
+  --docker-username=user \
+  --docker-password=password \
+  --docker-email=user@example.com
+```
+
+If the secret has been generated correctly you should see the following output:
+
+```console
+$ kubectl get secret kubeless-registry-credentials --output="jsonpath={.data.\.dockerconfigjson}" | base64 -d
+
+{"auths":{"https://index.docker.io/v1/":{"username":"user","password":"password","email":"user@example.com","auth":"dGVfdDpwYZNz"}}}
+```
+
+ 2. Enable the build step in the Kubeless configuration. If you have already deploy Kubeless you can enable it editing the configmap. You will need to set the property `enable-build-step: "false"` to `"true"`:
+
+ ```console
+ kubectl edit configmaps -n kubeless kubeless-config
+ ```
+
+ 3. Once the build step is enabled you need to restart the controller in order for the changes to take effect:
+
+ ```console
+ kubectl delete pod -n kubeless -l kubeless=controller
+ ```
+
+Once the secret is available and the build step is enabled Kubeless will automatically start building function images.
+
+## Build process
+
+The following diagram represents the building process:
+
+![Build Process](./img/build-process.png)
+
+When a new function is created the Kubeless Controller generates two items:
+ 
+ - A [Kubernetes job](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/) that will use the registry credentials to push a new image under the `user` repository. It will use the checksum (SHA256) of the function specification as tag so any change in the function will generate a different image.
+ - A Pod to run the function. This pod will wait until the previus job finishes in order to pull the function image.
+
+## Known limitations
+
+ - It is only possible to use a single registry to pull images and push them so if the build system is used with a registry different than https://index.docker.io/v1/ (the official one) the images present in the Kubeless ConfigMap should be copied to the new registry.
+ - Base images are not currently cached, that means that every time a new build is triggered it will download the base image.
+ 

examples/Makefile

@@ -34,6 +34,8 @@ get-python-deps-update:
 
 get-python-deps-update-verify:
 	$(eval pod := $(shell kubectl get pod -l function=get-python-deps -o go-template -o custom-columns=:metadata.name --no-headers=true))
+	echo "Checking updated deps of $(pod)"
+	kubectl exec -it $(pod) pip freeze
 	kubectl exec -it $(pod) pip freeze | grep -q "twitter=="
 
 get-python-34:
@@ -204,13 +206,13 @@ get-dotnetcore-verify:
 	kubeless function call get-dotnetcore |egrep hello.world
 
 custom-get-python:
-	kubeless function deploy --runtime-image kubeless/get-python-example@sha256:a922942597ce617adbe808b9f0cdc3cf7ff987a1277adf0233dd47be5d85082a custom-get-python
+	kubeless function deploy --runtime-image kubeless/get-python-example@sha256:af01f42956ca9981bc4ccec0c7df553539a44630fb87e00e562091a95e75dd21 custom-get-python
 
 custom-get-python-verify:
 	kubeless function call custom-get-python |egrep hello.world
 
 custom-get-python-update:
-	kubeless function update --runtime-image kubeless/get-python-example@sha256:d2ca4ab086564afbac6d30c29614f1623ddb9163b818537742c42fd785fcf2ce custom-get-python
+	kubeless function update --runtime-image kubeless/get-python-example@sha256:2cbc81d6412be01596bb5d8f3541a8e3e30ae6a8634122764bd32cff400dac3d custom-get-python
 
 custom-get-python-update-verify:
 	kubeless function call custom-get-python |egrep hello.world.updated

examples/python/Dockerfile

@@ -3,4 +3,5 @@ FROM kubeless/python@sha256:ba948a6783b93d75037b7b1806a3925d441401ae6fba18282f71
 ENV FUNC_HANDLER=foo \
     MOD_NAME=helloget
 ADD helloget.py /
+RUN mkdir -p /kubeless/
 ENTRYPOINT [ "bash", "-c", "mv /helloget.py /kubeless/ && python /kubeless.py"]

kubeless-rbac.jsonnet

@@ -21,15 +21,21 @@ local controller_roles = [
     resources: ["pods"],
     verbs: ["list", "delete"],
   },
+  {
+    apiGroups: [""],
+    resources: ["secrets"],
+    resourceNames: ["kubeless-registry-credentials"],
+    verbs: ["get"],
+  },
   {
     apiGroups: ["kubeless.io"],
     resources: ["functions", "kafkatriggers", "httptriggers", "cronjobtriggers"],
     verbs: ["get", "list", "watch", "update"],
   },
   {
     apiGroups: ["batch"],
-    resources: ["cronjobs"],
-    verbs: ["create", "get", "delete", "list", "update", "patch"],
+    resources: ["cronjobs", "jobs"],
+    verbs: ["create", "get", "delete", "deletecollection", "list", "update", "patch"],
   },
   {
     apiGroups: ["autoscaling"],

kubeless.jsonnet

@@ -156,7 +156,9 @@ local kubelessConfig  = configMap.default("kubeless-config", namespace) +
     configMap.data({"ingress-enabled": "false"}) +
     configMap.data({"service-type": "ClusterIP"})+
     configMap.data({"deployment": std.toString(deploymentConfig)})+
-    configMap.data({"runtime-images": std.toString(runtime_images)});
+    configMap.data({"runtime-images": std.toString(runtime_images)})+
+    configMap.data({"enable-build-step": "false"})+
+    configMap.data({"builder-image": "andresmgot/kubeless-function-image-builder:latest"});
 
 {
   controllerAccount: k.util.prune(controllerAccount),

pkg/controller/function_controller.go

@@ -17,7 +17,9 @@ limitations under the License.
 package controller
 
 import (
+	"crypto/sha256"
 	"fmt"
+	"net/url"
 	"time"
 
 	monitoringv1alpha1 "github.com/coreos/prometheus-operator/pkg/client/monitoring/v1alpha1"
@@ -39,6 +41,7 @@ import (
 	"github.com/kubeless/kubeless/pkg/client/clientset/versioned"
 	kv1beta1 "github.com/kubeless/kubeless/pkg/client/informers/externalversions/kubeless/v1beta1"
 	"github.com/kubeless/kubeless/pkg/langruntime"
+	"github.com/kubeless/kubeless/pkg/registry"
 	"github.com/kubeless/kubeless/pkg/utils"
 )
 
@@ -251,6 +254,42 @@ func (c *FunctionController) processItem(key string) error {
 	return nil
 }
 
+// startImageBuildJob creates (if necessary) a job that will build an image for the given function
+// returns the name of the image, a boolean indicating if the build job has been created and an error
+func (c *FunctionController) startImageBuildJob(funcObj *kubelessApi.Function, or []metav1.OwnerReference) (string, bool, error) {
+	imagePullSecret, err := c.clientset.CoreV1().Secrets(funcObj.ObjectMeta.Namespace).Get("kubeless-registry-credentials", metav1.GetOptions{})
+	if err != nil {
+		return "", false, fmt.Errorf("Unable to locate registry credentials to build function image: %v", err)
+	}
+	reg, err := registry.New(*imagePullSecret)
+	if err != nil {
+		return "", false, fmt.Errorf("Unable to retrieve registry information: %v", err)
+	}
+	// Use function content and deps as tag (digested)
+	tag := fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("%v%v", funcObj.Spec.Function, funcObj.Spec.Deps))))
+	imageName := fmt.Sprintf("%s/%s", reg.Creds.Username, funcObj.ObjectMeta.Name)
+	// Check if image already exists
+	exists, err := reg.ImageExists(imageName, tag)
+	if err != nil {
+		return "", false, fmt.Errorf("Unable to check is target image exists: %v", err)
+	}
+	image := fmt.Sprintf("%s:%s", imageName, tag)
+	if !exists {
+		regURL, err := url.Parse(reg.Endpoint)
+		if err != nil {
+			return "", false, fmt.Errorf("Unable to parse registry URL: %v", err)
+		}
+		err = utils.EnsureFuncImage(c.clientset, funcObj, c.langRuntime, or, imageName, tag, c.config.Data["builder-image"], regURL.Host, imagePullSecret.Name)
+		if err != nil {
+			return "", false, fmt.Errorf("Unable to create image build job: %v", err)
+		}
+	} else {
+		// Image already exists
+		return image, false, nil
+	}
+	return image, true, nil
+}
+
 // ensureK8sResources creates/updates k8s objects (deploy, svc, configmap) for the function
 func (c *FunctionController) ensureK8sResources(funcObj *kubelessApi.Function) error {
 	if len(funcObj.ObjectMeta.Labels) == 0 {
@@ -287,7 +326,30 @@ func (c *FunctionController) ensureK8sResources(funcObj *kubelessApi.Function) e
 		return err
 	}
 
-	err = utils.EnsureFuncDeployment(c.clientset, funcObj, or, c.langRuntime)
+	prebuiltImage := ""
+	if len(funcObj.Spec.Deployment.Spec.Template.Spec.Containers) > 0 && funcObj.Spec.Deployment.Spec.Template.Spec.Containers[0].Image != "" {
+		prebuiltImage = funcObj.Spec.Deployment.Spec.Template.Spec.Containers[0].Image
+	}
+	// Skip image build step if using a custom runtime
+	if prebuiltImage == "" {
+		if c.config.Data["enable-build-step"] == "true" {
+			var isBuilding bool
+			prebuiltImage, isBuilding, err = c.startImageBuildJob(funcObj, or)
+			if err != nil {
+				logrus.Errorf("Unable to build function: %v", err)
+			} else {
+				if isBuilding {
+					logrus.Infof("Started build process for function %s", funcObj.ObjectMeta.Name)
+				} else {
+					logrus.Infof("Found existing image %s", prebuiltImage)
+				}
+			}
+		}
+	} else {
+		logrus.Infof("Skipping image-build step for %s", funcObj.ObjectMeta.Name)
+	}
+
+	err = utils.EnsureFuncDeployment(c.clientset, funcObj, or, c.langRuntime, prebuiltImage)
 	if err != nil {
 		return err
 	}
@@ -358,6 +420,14 @@ func (c *FunctionController) deleteK8sResources(ns, name string) error {
 		return err
 	}
 
+	// delete build job
+	err = c.clientset.BatchV1().Jobs(ns).DeleteCollection(&metav1.DeleteOptions{}, metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("created-by=kubeless,function=%s", name),
+	})
+	if err != nil && !k8sErrors.IsNotFound(err) {
+		return err
+	}
+
 	return nil
 }