run builds using a persistent buildkit daemon

deployment/Dockerfile

@@ -1,11 +1,18 @@
+###############################################################
+FROM moby/buildkit:v0.7.2-rootless AS buildkit
+
 # build-env ###################################################
 FROM golang:1.15.5-buster AS build-env
 
+# skopeo
 RUN echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
 RUN curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_10/Release.key | apt-key add -
 RUN apt-get -y update
 RUN apt-get -y install skopeo
 
+# buildctl
+COPY --from=buildkit /usr/bin/buildctl /usr/local/bin/buildctl
+
 WORKDIR /app
 
 RUN apt-get update
@@ -30,8 +37,14 @@ RUN go install -ldflags="-X main.gitRef=${SOURCE_BRANCH} -X main.gitHash=${SOURC
 
 ###############################################################
 FROM alpine:3.12.1 AS prod
-RUN apk add --no-cache ca-certificates
+
+# skopeo
 RUN apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/community skopeo
+
+# buildctl
+COPY --from=buildkit /usr/bin/buildctl /usr/local/bin/buildctl
+
+RUN apk add --no-cache ca-certificates
 WORKDIR /root/
 COPY --from=build-env /go/bin/wedding /usr/local/bin/wedding
-ENTRYPOINT [ "./wedding", "server" ]
+ENTRYPOINT [ "wedding", "server" ]

deployment/kubernetes.yaml

@@ -48,10 +48,22 @@ spec:
             - name: minio
               mountPath: "/secret/minio"
               readOnly: true
+        - name: buildkit
+          image: moby/buildkit:v0.7.2-rootless
+          args:
+            - --addr=tcp://127.0.0.1:1234
+            - --oci-worker-no-process-sandbox
+          volumeMounts:
+            - name: buildkitd-config
+              mountPath: "/home/user/.config/buildkit"
+              readOnly: true
       volumes:
         - name: minio
           secret:
             secretName: minio
+        - name: buildkitd-config
+          configMap:
+            name: buildkitd-config
 ---
 apiVersion: v1
 kind: ServiceAccount

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/aws/aws-sdk-go v1.35.31
 	github.com/gorilla/mux v1.8.0
 	github.com/urfave/cli/v2 v2.3.0
+	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
 	k8s.io/api v0.19.4
 	k8s.io/apimachinery v0.19.4
 	k8s.io/client-go v0.19.4

go.sum

@@ -27,10 +27,6 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/aws/aws-sdk-go v1.35.23 h1:SCP0d0XvyJTDmfnHEQPvBaYi3kea1VNUo7uQmkVgFts=
-github.com/aws/aws-sdk-go v1.35.23/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
-github.com/aws/aws-sdk-go v1.35.29 h1:1kYnwrWTp2e+lI9yYFaDo7OFaLug8yXC6Qdj+u8451Q=
-github.com/aws/aws-sdk-go v1.35.29/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
 github.com/aws/aws-sdk-go v1.35.31 h1:6tlaYq4Q311qfhft/fIaND33XI27aW3zIdictcHxifE=
 github.com/aws/aws-sdk-go v1.35.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -202,7 +198,6 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
@@ -219,6 +214,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -326,16 +322,10 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-k8s.io/api v0.19.3 h1:GN6ntFnv44Vptj/b+OnMW7FmzkpDoIDLZRvKX3XH9aU=
-k8s.io/api v0.19.3/go.mod h1:VF+5FT1B74Pw3KxMdKyinLo+zynBaMBiAfGMuldcNDs=
 k8s.io/api v0.19.4 h1:I+1I4cgJYuCDgiLNjKx7SLmIbwgj9w7N7Zr5vSIdwpo=
 k8s.io/api v0.19.4/go.mod h1:SbtJ2aHCItirzdJ36YslycFNzWADYH3tgOhvBEFtZAk=
-k8s.io/apimachinery v0.19.3 h1:bpIQXlKjB4cB/oNpnNnV+BybGPR7iP5oYpsOTEJ4hgc=
-k8s.io/apimachinery v0.19.3/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA=
 k8s.io/apimachinery v0.19.4 h1:+ZoddM7nbzrDCp0T3SWnyxqf8cbWPT2fkZImoyvHUG0=
 k8s.io/apimachinery v0.19.4/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA=
-k8s.io/client-go v0.19.3 h1:ctqR1nQ52NUs6LpI0w+a5U+xjYwflFwA13OJKcicMxg=
-k8s.io/client-go v0.19.3/go.mod h1:+eEMktZM+MG0KO+PTkci8xnbCZHvj9TqR6Q1XDUIJOM=
 k8s.io/client-go v0.19.4 h1:85D3mDNoLF+xqpyE9Dh/OtrJDyJrSRKkHmDXIbEzer8=
 k8s.io/client-go v0.19.4/go.mod h1:ZrEy7+wj9PjH5VMBCuu/BDlvtUAku0oVFk4MmnW9mWA=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=

pkg/build.go

@@ -6,8 +6,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
+	"os"
+	"os/exec"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -16,6 +19,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"golang.org/x/sync/semaphore"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -48,30 +52,14 @@ type ObjectStore struct {
 }
 
 func (s Service) build(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
-
 	cfg, err := buildParameters(r)
 	if err != nil {
 		printBuildHelpText(w, err)
 		return
 	}
 
-	err = s.objectStore.storeContext(ctx, r.Body, cfg)
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write([]byte(fmt.Sprintf("store context: %v", err)))
-		log.Printf("execute build: %v", err)
-		return
-	}
-	defer func() {
-		s.objectStore.deleteContext(ctx, cfg)
-	}()
-
-	err = s.executeBuild(ctx, cfg, w)
-	if err != nil {
-		log.Printf("execute build: %v", err)
-		return
-	}
+	//s.buildInKubernetes(w, r, cfg)
+	buildLocally(w, r, cfg)
 }
 
 func buildParameters(r *http.Request) (*buildConfig, error) {
@@ -199,6 +187,27 @@ func printBuildHelpText(w http.ResponseWriter, err error) {
 	}
 }
 
+func (s Service) buildInKubernetes(w http.ResponseWriter, r *http.Request, cfg *buildConfig) {
+	ctx := r.Context()
+
+	err := s.objectStore.storeContext(ctx, r.Body, cfg)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(fmt.Sprintf("store context: %v", err)))
+		log.Printf("execute build: %v", err)
+		return
+	}
+	defer func() {
+		s.objectStore.deleteContext(ctx, cfg)
+	}()
+
+	err = s.executeBuild(ctx, cfg, w)
+	if err != nil {
+		log.Printf("execute build: %v", err)
+		return
+	}
+}
+
 func (o ObjectStore) storeContext(ctx context.Context, r io.Reader, cfg *buildConfig) error {
 	path := fmt.Sprintf("%d.tar", time.Now().UnixNano())
 	cfg.contextFilePath = path
@@ -413,6 +422,139 @@ buildctl-daemonless.sh \
 	return nil
 }
 
+func buildLocally(w http.ResponseWriter, r *http.Request, cfg *buildConfig) {
+	err := buildLocallyError(w, r, cfg)
+	if err != nil {
+		log.Printf("execute build: %v", err)
+	}
+}
+
+var sem = semaphore.NewWeighted(1)
+
+func buildLocallyError(w http.ResponseWriter, r *http.Request, cfg *buildConfig) error {
+	err := sem.Acquire(r.Context(), 1)
+	if err != nil {
+		return fmt.Errorf("acquire build semaphore: %v", err)
+	}
+	defer sem.Release(1)
+
+	o := &output{w: w}
+	d := &digestParser{w: o}
+
+	defer os.RemoveAll("/root/context")
+
+	script := `
+set -euo pipefail
+mkdir /root/context
+cd /root/context
+tar -xf -
+`
+	cmd := exec.CommandContext(
+		r.Context(),
+		"timeout",
+		strconv.Itoa(int(MaxExecutionTime/time.Second)),
+		"bash",
+		"-c",
+		script,
+	)
+	cmd.Stdout = w
+	cmd.Stderr = w
+	cmd.Stdin = r.Body
+
+	err = cmd.Run()
+	if err != nil {
+		o.Errorf("extract context: %v", err)
+		return fmt.Errorf("extract context: %v", err)
+	}
+
+	err = os.MkdirAll("/root/.docker/", os.ModePerm)
+	if err != nil {
+		o.Errorf("prepare docker config directory: %v", err)
+		return fmt.Errorf("prepare docker config directory: %v", err)
+	}
+
+	err = ioutil.WriteFile("/root/.docker/config.json", []byte(cfg.registryAuth.mustToJSON()), os.ModePerm)
+	if err != nil {
+		o.Errorf("write docker auth config: %v", err)
+		return fmt.Errorf("write docker auth config: %v", err)
+	}
+	defer os.Remove("/root/.docker/config.json")
+
+	imageNames := ""
+	for idx, tag := range cfg.tags {
+		if idx != 0 {
+			imageNames += ","
+		}
+		imageNames += fmt.Sprintf("wedding-registry:5000/images/%s", tag)
+	}
+
+	destination := "--output type=image,push=true,name=wedding-registry:5000/digests"
+	if imageNames != "" {
+		destination = fmt.Sprintf(`--output type=image,push=true,\"name=%s\"`, imageNames)
+	}
+
+	dockerfileName := filepath.Base(cfg.dockerfile)
+	dockerfileDir := filepath.Dir(cfg.dockerfile)
+
+	target := ""
+	if cfg.target != "" {
+		target = fmt.Sprintf("--opt target=%s", cfg.target)
+	}
+
+	buildargs := ""
+	for k, v := range cfg.buildArgs {
+		buildargs += fmt.Sprintf("--opt build-arg:%s='%s' ", k, v)
+	}
+
+	labels := ""
+	for k, v := range cfg.labels {
+		buildargs += fmt.Sprintf("--opt label:%s='%s' ", k, v)
+	}
+
+	script = fmt.Sprintf(`
+set -exuo pipefail
+cd /root/context
+buildctl \
+--addr tcp://127.0.0.1:1234 \
+ build \
+ --frontend dockerfile.v0 \
+ --local context=. \
+ --local dockerfile=%s \
+ --opt filename=%s \
+ %s \
+ %s \
+ %s \
+ %s \
+ --export-cache=type=registry,ref=wedding-registry:5000/cache-repo,mode=max \
+ --import-cache=type=registry,ref=wedding-registry:5000/cache-repo
+`, dockerfileDir, dockerfileName, buildargs, labels, target, destination)
+
+	cmd = exec.CommandContext(
+		r.Context(),
+		"timeout",
+		strconv.Itoa(int(MaxExecutionTime/time.Second)),
+		"bash",
+		"-c",
+		script,
+	)
+	cmd.Stdout = d
+	cmd.Stderr = d
+
+	err = cmd.Run()
+	if err != nil {
+		o.Errorf("execute build: %v", err)
+		return fmt.Errorf("execute build: %v", err)
+	}
+
+	err = d.publish(w)
+	if err != nil {
+		o.Errorf("publish ID: %v", err)
+		return fmt.Errorf("publish ID: %v", err)
+	}
+
+	return nil
+}
+
 type digestParser struct {
 	buf bytes.Buffer
 	w   io.Writer