only execute tasks via kubernetes

Tiltfile

@@ -1,4 +1,5 @@
 disable_snapshots()
+analytics_settings(enable=False)
 allow_k8s_contexts(['test', 'ci'])
 
 load('ext://min_tilt_version', 'min_tilt_version')

deployment/Dockerfile

@@ -1,18 +1,6 @@
-###############################################################
-FROM moby/buildkit:v0.8.3-rootless AS buildkit
-
 # build-env ###################################################
 FROM golang:1.16.4-buster AS build-env
 
-# skopeo
-RUN echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
-RUN curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_10/Release.key | apt-key add -
-RUN apt-get -y update
-RUN apt-get -y install skopeo
-
-# buildctl
-COPY --from=buildkit /usr/bin/buildctl /usr/local/bin/buildctl
-
 WORKDIR /app
 
 RUN apt-get update
@@ -38,12 +26,6 @@ RUN go install -ldflags="-X main.gitRef=${SOURCE_BRANCH} -X main.gitHash=${SOURC
 ###############################################################
 FROM alpine:3.13.5 AS prod
 
-# skopeo
-RUN apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/community skopeo
-
-# buildctl
-COPY --from=buildkit /usr/bin/buildctl /usr/local/bin/buildctl
-
 RUN apk add --no-cache ca-certificates
 WORKDIR /root/
 COPY --from=build-env /go/bin/wedding /usr/local/bin/wedding

deployment/kubernetes.yaml

@@ -55,29 +55,10 @@ spec:
             limits:
               memory: "200Mi"
               cpu: "200m"
-        - name: buildkit
-          image: moby/buildkit:v0.8.1-rootless
-          args:
-            - --addr=tcp://127.0.0.1:1234
-            - --oci-worker-no-process-sandbox
-          volumeMounts:
-            - name: buildkitd-config
-              mountPath: "/home/user/.config/buildkit"
-              readOnly: true
-          resources:
-            requests:
-              memory: "2148Mi"
-              cpu: "500m"
-            limits:
-              memory: "2148Mi"
-              cpu: "1000m"
       volumes:
         - name: minio
           secret:
             secretName: wedding-minio
-        - name: buildkitd-config
-          configMap:
-            name: buildkitd-config
 ---
 apiVersion: v1
 kind: ServiceAccount

pkg/build.go

@@ -6,11 +6,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net/http"
-	"os"
-	"os/exec"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -57,26 +54,7 @@ func (s Service) build(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	scheduler := s.buildInKubernetes
-	if cfg.fitsLocally() {
-		ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
-		defer cancel()
-
-		err = semBuild.Acquire(ctx, 1)
-		if err == nil {
-			log.Printf("build locally %v", cfg.tags)
-			defer semBuild.Release(1)
-			scheduler = buildLocally
-		} else {
-			log.Printf("build scheduled %v", cfg.tags)
-		}
-	}
-
-	scheduler(w, r, cfg)
-}
-
-func (c buildConfig) fitsLocally() bool {
-	return c.cpuMilliseconds <= 1000 && c.memoryBytes <= 1024*1024*1024*2
+	s.buildInKubernetes(w, r, cfg)
 }
 
 func buildParameters(r *http.Request) (*buildConfig, error) {
@@ -435,130 +413,6 @@ buildctl-daemonless.sh \
 	return nil
 }
 
-func buildLocally(w http.ResponseWriter, r *http.Request, cfg *buildConfig) {
-	o := &output{w: w}
-	d := &digestParser{w: o}
-
-	err := buildLocallyError(r.Context(), d, r.Body, cfg)
-	if err != nil {
-		log.Printf("execute build: %v", err)
-		o.Errorf("execute build: %v", err)
-		return
-	}
-
-	err = d.publish(w)
-	if err != nil {
-		log.Printf("publish ID: %v", err)
-		o.Errorf("publish ID: %v", err)
-		return
-	}
-}
-
-func buildLocallyError(ctx context.Context, w io.Writer, r io.Reader, cfg *buildConfig) error {
-	defer os.RemoveAll("/root/context")
-
-	script := `
-set -euo pipefail
-mkdir /root/context
-cd /root/context
-tar -xf -
-`
-	cmd := exec.CommandContext(
-		ctx,
-		"timeout",
-		strconv.Itoa(int(MaxExecutionTime/time.Second)),
-		"bash",
-		"-c",
-		script,
-	)
-	cmd.Stdout = w
-	cmd.Stderr = w
-	cmd.Stdin = r
-
-	err := cmd.Run()
-	if err != nil {
-		return fmt.Errorf("extract context: %v", err)
-	}
-
-	err = os.MkdirAll("/root/.docker/", os.ModePerm)
-	if err != nil {
-		return fmt.Errorf("prepare docker config directory: %v", err)
-	}
-
-	err = ioutil.WriteFile("/root/.docker/config.json", []byte(cfg.registryAuth.mustToJSON()), os.ModePerm)
-	if err != nil {
-		return fmt.Errorf("write docker auth config: %v", err)
-	}
-	defer os.Remove("/root/.docker/config.json")
-
-	imageNames := ""
-	for idx, tag := range cfg.tags {
-		if idx != 0 {
-			imageNames += ","
-		}
-		imageNames += fmt.Sprintf("wedding-registry:5000/images/%s", tag)
-	}
-
-	destination := "--output type=image,push=true,name=wedding-registry:5000/digests"
-	if imageNames != "" {
-		destination = fmt.Sprintf(`--output type=image,push=true,\"name=%s\"`, imageNames)
-	}
-
-	dockerfileName := filepath.Base(cfg.dockerfile)
-	dockerfileDir := filepath.Dir(cfg.dockerfile)
-
-	target := ""
-	if cfg.target != "" {
-		target = fmt.Sprintf("--opt target=%s", cfg.target)
-	}
-
-	buildargs := ""
-	for k, v := range cfg.buildArgs {
-		buildargs += fmt.Sprintf("--opt build-arg:%s='%s' ", k, v)
-	}
-
-	labels := ""
-	for k, v := range cfg.labels {
-		buildargs += fmt.Sprintf("--opt label:%s='%s' ", k, v)
-	}
-
-	script = fmt.Sprintf(`
-set -exuo pipefail
-cd /root/context
-buildctl \
---addr tcp://127.0.0.1:1234 \
- build \
- --frontend dockerfile.v0 \
- --local context=. \
- --local dockerfile=%s \
- --opt filename=%s \
- %s \
- %s \
- %s \
- %s \
- --export-cache=type=registry,ref=wedding-registry:5000/cache-repo,mode=max \
- --import-cache=type=registry,ref=wedding-registry:5000/cache-repo
-`, dockerfileDir, dockerfileName, buildargs, labels, target, destination)
-
-	cmd = exec.CommandContext(
-		ctx,
-		"timeout",
-		strconv.Itoa(int(MaxExecutionTime/time.Second)),
-		"bash",
-		"-c",
-		script,
-	)
-	cmd.Stdout = w
-	cmd.Stderr = w
-
-	err = cmd.Run()
-	if err != nil {
-		return fmt.Errorf("execute build: %v", err)
-	}
-
-	return nil
-}
-
 type digestParser struct {
 	buf bytes.Buffer
 	w   io.Writer

pkg/inspect.go

@@ -2,13 +2,11 @@ package wedding
 
 import (
 	"bytes"
-	"context"
 	"fmt"
 	"io"
 	"log"
 	"math/rand"
 	"net/http"
-	"time"
 
 	"github.com/gorilla/mux"
 )
@@ -25,21 +23,8 @@ skopeo inspect dir://%s
 rm -r %s
 `, randomID, image, randomID, randomID, randomID)
 
-	ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
-	defer cancel()
-
-	scheduler := s.scheduleInKubernetes
-	err := semSkopeo.Acquire(ctx, 1)
-	if err == nil {
-		log.Printf("inspect locally %s", vars["name"])
-		defer semSkopeo.Release(1)
-		scheduler = scheduleLocal
-	} else {
-		log.Printf("inspect scheduled %s", vars["name"])
-	}
-
 	o := &bytes.Buffer{}
-	err = scheduler(r.Context(), o, "inspect", script, "")
+	err := s.scheduleInKubernetes(r.Context(), o, "inspect", script, "")
 	if err != nil {
 		log.Printf("execute inspect: %v", err)
 		w.WriteHeader(http.StatusNotFound)

pkg/kubernetes.go

@@ -5,11 +5,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
-	"os/exec"
-	"path/filepath"
 	"strconv"
 	"time"
 
@@ -18,41 +15,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func scheduleLocal(ctx context.Context, w io.Writer, processName, script, dockerJSON string) error {
-	tmpHome, err := ioutil.TempDir("", "docker-secret")
-	if err != nil {
-		return fmt.Errorf("create tempdir for docker secret: %v", err)
-	}
-	defer os.RemoveAll(tmpHome)
-
-	if dockerJSON != "" {
-		err = os.Mkdir(filepath.Join(tmpHome, ".docker"), os.ModePerm)
-		if err != nil {
-			return fmt.Errorf("create .docker directory for docker secret: %v", err)
-		}
-
-		dockerConfigJSON := filepath.Join(tmpHome, ".docker", "config.json")
-		err = ioutil.WriteFile(dockerConfigJSON, []byte(dockerJSON), os.ModePerm)
-		if err != nil {
-			return fmt.Errorf("write docker secret: %v", err)
-		}
-	}
-
-	cmd := exec.CommandContext(
-		ctx,
-		"timeout",
-		strconv.Itoa(int(MaxExecutionTime/time.Second)),
-		"bash",
-		"-c",
-		script,
-	)
-	cmd.Stdout = w
-	cmd.Stderr = w
-	cmd.Env = append(cmd.Env, fmt.Sprintf("HOME=%s", tmpHome))
-
-	return cmd.Run()
-}
-
 func (s Service) scheduleInKubernetes(ctx context.Context, w io.Writer, processName, script, dockerJSON string) error {
 	pod := &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{

pkg/pull.go

@@ -1,11 +1,9 @@
 package wedding
 
 import (
-	"context"
 	"fmt"
 	"log"
 	"net/http"
-	"time"
 )
 
 func (s Service) pullImage(w http.ResponseWriter, r *http.Request) {
@@ -62,21 +60,8 @@ func (s Service) pullImage(w http.ResponseWriter, r *http.Request) {
 
 	script := fmt.Sprintf(`skopeo copy --retry-times 3 --dest-tls-verify=false docker://%s docker://%s`, from, to)
 
-	ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
-	defer cancel()
-
-	scheduler := s.scheduleInKubernetes
-	err = semSkopeo.Acquire(ctx, 1)
-	if err == nil {
-		log.Printf("pull locally %s", from)
-		defer semSkopeo.Release(1)
-		scheduler = scheduleLocal
-	} else {
-		log.Printf("pull scheduled %s", from)
-	}
-
 	o := &output{w: w}
-	err = scheduler(r.Context(), o, "pull", script, dockerCfg.mustToJSON())
+	err = s.scheduleInKubernetes(r.Context(), o, "pull", script, dockerCfg.mustToJSON())
 	if err != nil {
 		log.Printf("execute pull: %v", err)
 		o.Errorf("execute pull: %v", err)

pkg/push.go

@@ -1,11 +1,9 @@
 package wedding
 
 import (
-	"context"
 	"fmt"
 	"log"
 	"net/http"
-	"time"
 
 	"github.com/gorilla/mux"
 )
@@ -28,21 +26,8 @@ func (s Service) pushImage(w http.ResponseWriter, r *http.Request) {
 	// TODO only use --dest-tls-verify=false for local registry
 	script := fmt.Sprintf(`skopeo copy --retry-times 3 --src-tls-verify=false --dest-tls-verify=false docker://%s docker://%s`, from, to)
 
-	ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
-	defer cancel()
-
-	scheduler := s.scheduleInKubernetes
-	err = semSkopeo.Acquire(ctx, 1)
-	if err == nil {
-		log.Printf("push locally %s", to)
-		defer semSkopeo.Release(1)
-		scheduler = scheduleLocal
-	} else {
-		log.Printf("push scheduled %s", to)
-	}
-
 	o := &output{w: w}
-	err = scheduler(r.Context(), o, "push", script, dockerCfg.mustToJSON())
+	err = s.scheduleInKubernetes(r.Context(), o, "push", script, dockerCfg.mustToJSON())
 	if err != nil {
 		log.Printf("execute push: %v", err)
 		o.Errorf("execute push: %v", err)