From 227f78f854985688cfec423f1b65ad81f56ad4a8 Mon Sep 17 00:00:00 2001 From: David Sauer Date: Sun, 23 May 2021 17:42:29 +0200 Subject: [PATCH] only execute tasks via kubernetes --- Tiltfile | 1 + deployment/Dockerfile | 18 ----- deployment/kubernetes.yaml | 19 ----- pkg/build.go | 148 +------------------------------------ pkg/inspect.go | 17 +---- pkg/kubernetes.go | 38 ---------- pkg/pull.go | 17 +---- pkg/push.go | 17 +---- pkg/tag.go | 17 +---- 9 files changed, 6 insertions(+), 286 deletions(-) diff --git a/Tiltfile b/Tiltfile index 08ae4f3e..41e447cd 100644 --- a/Tiltfile +++ b/Tiltfile @@ -1,4 +1,5 @@ disable_snapshots() +analytics_settings(enable=False) allow_k8s_contexts(['test', 'ci']) load('ext://min_tilt_version', 'min_tilt_version') diff --git a/deployment/Dockerfile b/deployment/Dockerfile index d05c83ed..95a3a58f 100644 --- a/deployment/Dockerfile +++ b/deployment/Dockerfile @@ -1,18 +1,6 @@ -############################################################### -FROM moby/buildkit:v0.8.3-rootless AS buildkit - # build-env ################################################### FROM golang:1.16.4-buster AS build-env -# skopeo -RUN echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list -RUN curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_10/Release.key | apt-key add - -RUN apt-get -y update -RUN apt-get -y install skopeo - -# buildctl -COPY --from=buildkit /usr/bin/buildctl /usr/local/bin/buildctl - WORKDIR /app RUN apt-get update @@ -38,12 +26,6 @@ RUN go install -ldflags="-X main.gitRef=${SOURCE_BRANCH} -X main.gitHash=${SOURC ############################################################### FROM alpine:3.13.5 AS prod -# skopeo -RUN apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/community skopeo - -# buildctl -COPY --from=buildkit /usr/bin/buildctl /usr/local/bin/buildctl - RUN apk add --no-cache ca-certificates WORKDIR /root/ COPY --from=build-env /go/bin/wedding /usr/local/bin/wedding diff --git a/deployment/kubernetes.yaml b/deployment/kubernetes.yaml index 72672d2b..ad48b6d5 100644 --- a/deployment/kubernetes.yaml +++ b/deployment/kubernetes.yaml @@ -55,29 +55,10 @@ spec: limits: memory: "200Mi" cpu: "200m" - - name: buildkit - image: moby/buildkit:v0.8.1-rootless - args: - - --addr=tcp://127.0.0.1:1234 - - --oci-worker-no-process-sandbox - volumeMounts: - - name: buildkitd-config - mountPath: "/home/user/.config/buildkit" - readOnly: true - resources: - requests: - memory: "2148Mi" - cpu: "500m" - limits: - memory: "2148Mi" - cpu: "1000m" volumes: - name: minio secret: secretName: wedding-minio - - name: buildkitd-config - configMap: - name: buildkitd-config --- apiVersion: v1 kind: ServiceAccount diff --git a/pkg/build.go b/pkg/build.go index 57069f78..2704259a 100644 --- a/pkg/build.go +++ b/pkg/build.go @@ -6,11 +6,8 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "log" "net/http" - "os" - "os/exec" "path/filepath" "regexp" "strconv" @@ -57,26 +54,7 @@ func (s Service) build(w http.ResponseWriter, r *http.Request) { return } - scheduler := s.buildInKubernetes - if cfg.fitsLocally() { - ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) - defer cancel() - - err = semBuild.Acquire(ctx, 1) - if err == nil { - log.Printf("build locally %v", cfg.tags) - defer semBuild.Release(1) - scheduler = buildLocally - } else { - log.Printf("build scheduled %v", cfg.tags) - } - } - - scheduler(w, r, cfg) -} - -func (c buildConfig) fitsLocally() bool { - return c.cpuMilliseconds <= 1000 && c.memoryBytes <= 1024*1024*1024*2 + s.buildInKubernetes(w, r, cfg) } func buildParameters(r *http.Request) (*buildConfig, error) { @@ -435,130 +413,6 @@ buildctl-daemonless.sh \ return nil } -func buildLocally(w http.ResponseWriter, r *http.Request, cfg *buildConfig) { - o := &output{w: w} - d := &digestParser{w: o} - - err := buildLocallyError(r.Context(), d, r.Body, cfg) - if err != nil { - log.Printf("execute build: %v", err) - o.Errorf("execute build: %v", err) - return - } - - err = d.publish(w) - if err != nil { - log.Printf("publish ID: %v", err) - o.Errorf("publish ID: %v", err) - return - } -} - -func buildLocallyError(ctx context.Context, w io.Writer, r io.Reader, cfg *buildConfig) error { - defer os.RemoveAll("/root/context") - - script := ` -set -euo pipefail -mkdir /root/context -cd /root/context -tar -xf - -` - cmd := exec.CommandContext( - ctx, - "timeout", - strconv.Itoa(int(MaxExecutionTime/time.Second)), - "bash", - "-c", - script, - ) - cmd.Stdout = w - cmd.Stderr = w - cmd.Stdin = r - - err := cmd.Run() - if err != nil { - return fmt.Errorf("extract context: %v", err) - } - - err = os.MkdirAll("/root/.docker/", os.ModePerm) - if err != nil { - return fmt.Errorf("prepare docker config directory: %v", err) - } - - err = ioutil.WriteFile("/root/.docker/config.json", []byte(cfg.registryAuth.mustToJSON()), os.ModePerm) - if err != nil { - return fmt.Errorf("write docker auth config: %v", err) - } - defer os.Remove("/root/.docker/config.json") - - imageNames := "" - for idx, tag := range cfg.tags { - if idx != 0 { - imageNames += "," - } - imageNames += fmt.Sprintf("wedding-registry:5000/images/%s", tag) - } - - destination := "--output type=image,push=true,name=wedding-registry:5000/digests" - if imageNames != "" { - destination = fmt.Sprintf(`--output type=image,push=true,\"name=%s\"`, imageNames) - } - - dockerfileName := filepath.Base(cfg.dockerfile) - dockerfileDir := filepath.Dir(cfg.dockerfile) - - target := "" - if cfg.target != "" { - target = fmt.Sprintf("--opt target=%s", cfg.target) - } - - buildargs := "" - for k, v := range cfg.buildArgs { - buildargs += fmt.Sprintf("--opt build-arg:%s='%s' ", k, v) - } - - labels := "" - for k, v := range cfg.labels { - buildargs += fmt.Sprintf("--opt label:%s='%s' ", k, v) - } - - script = fmt.Sprintf(` -set -exuo pipefail -cd /root/context -buildctl \ ---addr tcp://127.0.0.1:1234 \ - build \ - --frontend dockerfile.v0 \ - --local context=. \ - --local dockerfile=%s \ - --opt filename=%s \ - %s \ - %s \ - %s \ - %s \ - --export-cache=type=registry,ref=wedding-registry:5000/cache-repo,mode=max \ - --import-cache=type=registry,ref=wedding-registry:5000/cache-repo -`, dockerfileDir, dockerfileName, buildargs, labels, target, destination) - - cmd = exec.CommandContext( - ctx, - "timeout", - strconv.Itoa(int(MaxExecutionTime/time.Second)), - "bash", - "-c", - script, - ) - cmd.Stdout = w - cmd.Stderr = w - - err = cmd.Run() - if err != nil { - return fmt.Errorf("execute build: %v", err) - } - - return nil -} - type digestParser struct { buf bytes.Buffer w io.Writer diff --git a/pkg/inspect.go b/pkg/inspect.go index 9014dbc0..3f55b9e9 100644 --- a/pkg/inspect.go +++ b/pkg/inspect.go @@ -2,13 +2,11 @@ package wedding import ( "bytes" - "context" "fmt" "io" "log" "math/rand" "net/http" - "time" "github.com/gorilla/mux" ) @@ -25,21 +23,8 @@ skopeo inspect dir://%s rm -r %s `, randomID, image, randomID, randomID, randomID) - ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) - defer cancel() - - scheduler := s.scheduleInKubernetes - err := semSkopeo.Acquire(ctx, 1) - if err == nil { - log.Printf("inspect locally %s", vars["name"]) - defer semSkopeo.Release(1) - scheduler = scheduleLocal - } else { - log.Printf("inspect scheduled %s", vars["name"]) - } - o := &bytes.Buffer{} - err = scheduler(r.Context(), o, "inspect", script, "") + err := s.scheduleInKubernetes(r.Context(), o, "inspect", script, "") if err != nil { log.Printf("execute inspect: %v", err) w.WriteHeader(http.StatusNotFound) diff --git a/pkg/kubernetes.go b/pkg/kubernetes.go index 1b1ce5d7..215f8fce 100644 --- a/pkg/kubernetes.go +++ b/pkg/kubernetes.go @@ -5,11 +5,8 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "log" "os" - "os/exec" - "path/filepath" "strconv" "time" @@ -18,41 +15,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func scheduleLocal(ctx context.Context, w io.Writer, processName, script, dockerJSON string) error { - tmpHome, err := ioutil.TempDir("", "docker-secret") - if err != nil { - return fmt.Errorf("create tempdir for docker secret: %v", err) - } - defer os.RemoveAll(tmpHome) - - if dockerJSON != "" { - err = os.Mkdir(filepath.Join(tmpHome, ".docker"), os.ModePerm) - if err != nil { - return fmt.Errorf("create .docker directory for docker secret: %v", err) - } - - dockerConfigJSON := filepath.Join(tmpHome, ".docker", "config.json") - err = ioutil.WriteFile(dockerConfigJSON, []byte(dockerJSON), os.ModePerm) - if err != nil { - return fmt.Errorf("write docker secret: %v", err) - } - } - - cmd := exec.CommandContext( - ctx, - "timeout", - strconv.Itoa(int(MaxExecutionTime/time.Second)), - "bash", - "-c", - script, - ) - cmd.Stdout = w - cmd.Stderr = w - cmd.Env = append(cmd.Env, fmt.Sprintf("HOME=%s", tmpHome)) - - return cmd.Run() -} - func (s Service) scheduleInKubernetes(ctx context.Context, w io.Writer, processName, script, dockerJSON string) error { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/pull.go b/pkg/pull.go index 9af5d7f9..e1f8f38a 100644 --- a/pkg/pull.go +++ b/pkg/pull.go @@ -1,11 +1,9 @@ package wedding import ( - "context" "fmt" "log" "net/http" - "time" ) func (s Service) pullImage(w http.ResponseWriter, r *http.Request) { @@ -62,21 +60,8 @@ func (s Service) pullImage(w http.ResponseWriter, r *http.Request) { script := fmt.Sprintf(`skopeo copy --retry-times 3 --dest-tls-verify=false docker://%s docker://%s`, from, to) - ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) - defer cancel() - - scheduler := s.scheduleInKubernetes - err = semSkopeo.Acquire(ctx, 1) - if err == nil { - log.Printf("pull locally %s", from) - defer semSkopeo.Release(1) - scheduler = scheduleLocal - } else { - log.Printf("pull scheduled %s", from) - } - o := &output{w: w} - err = scheduler(r.Context(), o, "pull", script, dockerCfg.mustToJSON()) + err = s.scheduleInKubernetes(r.Context(), o, "pull", script, dockerCfg.mustToJSON()) if err != nil { log.Printf("execute pull: %v", err) o.Errorf("execute pull: %v", err) diff --git a/pkg/push.go b/pkg/push.go index 4143ae47..794aa358 100644 --- a/pkg/push.go +++ b/pkg/push.go @@ -1,11 +1,9 @@ package wedding import ( - "context" "fmt" "log" "net/http" - "time" "github.com/gorilla/mux" ) @@ -28,21 +26,8 @@ func (s Service) pushImage(w http.ResponseWriter, r *http.Request) { // TODO only use --dest-tls-verify=false for local registry script := fmt.Sprintf(`skopeo copy --retry-times 3 --src-tls-verify=false --dest-tls-verify=false docker://%s docker://%s`, from, to) - ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) - defer cancel() - - scheduler := s.scheduleInKubernetes - err = semSkopeo.Acquire(ctx, 1) - if err == nil { - log.Printf("push locally %s", to) - defer semSkopeo.Release(1) - scheduler = scheduleLocal - } else { - log.Printf("push scheduled %s", to) - } - o := &output{w: w} - err = scheduler(r.Context(), o, "push", script, dockerCfg.mustToJSON()) + err = s.scheduleInKubernetes(r.Context(), o, "push", script, dockerCfg.mustToJSON()) if err != nil { log.Printf("execute push: %v", err) o.Errorf("execute push: %v", err) diff --git a/pkg/tag.go b/pkg/tag.go index 14c1e6ac..537b4185 100644 --- a/pkg/tag.go +++ b/pkg/tag.go @@ -2,14 +2,12 @@ package wedding import ( "bytes" - "context" "fmt" "io" "log" "net/http" "regexp" "strings" - "time" "github.com/gorilla/mux" ) @@ -36,21 +34,8 @@ func (s Service) tagImage(w http.ResponseWriter, r *http.Request) { script := fmt.Sprintf(`skopeo copy --retry-times 3 --src-tls-verify=false --dest-tls-verify=false docker://%s docker://%s`, from, to) - ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) - defer cancel() - - scheduler := s.scheduleInKubernetes - err := semSkopeo.Acquire(ctx, 1) - if err == nil { - log.Printf("tag locally %s to %s", tag, to) - defer semSkopeo.Release(1) - scheduler = scheduleLocal - } else { - log.Printf("tag scheduled %s to %s", tag, to) - } - o := &bytes.Buffer{} - err = scheduler(r.Context(), o, "tag", script, "") + err := s.scheduleInKubernetes(r.Context(), o, "tag", script, "") if err != nil { log.Printf("execute tag: %v", err) w.WriteHeader(http.StatusInternalServerError)