From a7b50b0afd044ce67cd48fdb6f84d4589ae664dd Mon Sep 17 00:00:00 2001 From: David Sauer Date: Fri, 13 Nov 2020 16:29:44 +0100 Subject: [PATCH] automaticly execute tests on code change --- Tiltfile | 21 +-- pkg/build.go | 7 +- pkg/kubernetes.go | 10 +- pkg/pull.go | 150 +++++++++++++++++++++ pkg/push.go | 2 +- pkg/registry_authentication.go | 2 - pkg/service.go | 19 +-- pkg/tag.go | 28 +++- services/wedding-registry.yaml | 5 + tests/Dockerfile | 9 ++ tests/Tiltfile | 15 +++ tests/docker-build.yaml | 27 ++++ tests/docker-pull-tag-push.yaml | 33 +++++ {use-case-1 => tests/docker}/Dockerfile | 0 tests/tilt-ci.yaml | 35 +++++ {use-case-3 => tests/tilt}/Dockerfile | 0 {use-case-3 => tests/tilt}/Makefile | 0 {use-case-3 => tests/tilt}/Tiltfile | 0 {use-case-3 => tests/tilt}/kubernetes.yaml | 0 19 files changed, 322 insertions(+), 41 deletions(-) create mode 100644 pkg/pull.go create mode 100644 tests/Dockerfile create mode 100644 tests/Tiltfile create mode 100644 tests/docker-build.yaml create mode 100644 tests/docker-pull-tag-push.yaml rename {use-case-1 => tests/docker}/Dockerfile (100%) create mode 100644 tests/tilt-ci.yaml rename {use-case-3 => tests/tilt}/Dockerfile (100%) rename {use-case-3 => tests/tilt}/Makefile (100%) rename {use-case-3 => tests/tilt}/Tiltfile (100%) rename {use-case-3 => tests/tilt}/kubernetes.yaml (100%) diff --git a/Tiltfile b/Tiltfile index bb648078..27025fd6 100644 --- a/Tiltfile +++ b/Tiltfile @@ -5,6 +5,7 @@ load('ext://min_tilt_version', 'min_tilt_version') min_tilt_version('0.15.0') # includes fix for auto_init+False with tilt ci include('./services/Tiltfile') +include('./tests/Tiltfile') k8s_yaml('deployment/kubernetes.yaml') @@ -56,23 +57,5 @@ else: k8s_resource( 'wedding', port_forwards=['12376:2376'], - resource_deps=['setup-s3-bucket'], -) - -local_resource ('docker build', - 'DOCKER_HOST=tcp://127.0.0.1:12376 docker build ./use-case-1', - resource_deps=['wedding'], -) - -local_resource ('docker pull tag push', - 'DOCKER_HOST=tcp://127.0.0.1:12376 docker pull alpine && \ - DOCKER_HOST=tcp://127.0.0.1:12376 docker tag alpine wedding-registry:5000/alpine-retag && \ - DOCKER_HOST=tcp://127.0.0.1:12376 docker push wedding-registry:5000/alpine-retag', - resource_deps=['wedding'], -) - -local_resource ('tilt ci', - 'cd use-case-3 && DOCKER_HOST=tcp://127.0.0.1:12376 tilt ci --port 0 && \ - tilt down', - resource_deps=['wedding'], + resource_deps=['setup-s3-bucket', 'wedding-registry'], ) diff --git a/pkg/build.go b/pkg/build.go index 94ad4928..7f02a637 100644 --- a/pkg/build.go +++ b/pkg/build.go @@ -193,7 +193,6 @@ func buildParameters(r *http.Request) (*buildConfig, error) { cfg.noCache = nocache == "1" // registry authentitation - log.Printf("X-Registry-Config: %v", r.Header.Get("X-Registry-Config")) dockerCfg, err := xRegistryConfig(r.Header.Get("X-Registry-Config")).toDockerConfig() if err != nil { return cfg, fmt.Errorf("extract registry config: %v", err) @@ -255,7 +254,7 @@ func (o ObjectStore) presignContext(cfg *buildConfig) (string, error) { Key: aws.String(cfg.contextFilePath), }) - url, err := objectRequest.Presign(time.Hour) + url, err := objectRequest.Presign(10 * time.Minute) if err != nil { return "", fmt.Errorf("presign GET %s: %v", cfg.contextFilePath, err) } @@ -321,7 +320,7 @@ func (s Service) executeBuild(ctx context.Context, cfg *buildConfig, w http.Resp output := "--output type=image,push=true,name=wedding-registry:5000/digests" if imageNames != "" { - output = fmt.Sprintf("--output type=image,push=true,name=\"%s\"", imageNames) + output = fmt.Sprintf("--output type=image,push=true,\"name=%s\"", imageNames) } // TODO add timeout for script @@ -362,8 +361,8 @@ buildctl-daemonless.sh \ Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "buildkit", Image: "moby/buildkit:v0.7.2-rootless", + Name: "buildkit", Command: []string{ "sh", "-c", diff --git a/pkg/kubernetes.go b/pkg/kubernetes.go index 91dcf57c..7a9f4e31 100644 --- a/pkg/kubernetes.go +++ b/pkg/kubernetes.go @@ -110,13 +110,21 @@ func (s Service) podStatus(ctx context.Context, podName string) (corev1.PodPhase return pod.Status.Phase, nil } +func message(w io.Writer, message string) error { + return flush(w, "message", message) +} + func stream(w io.Writer, message string) error { + return flush(w, "stream", message) +} + +func flush(w io.Writer, kind, message string) error { b, err := json.Marshal(message) if err != nil { panic(err) // encode a string to json should not fail } - _, err = w.Write([]byte(fmt.Sprintf(`{"stream": %s}`, b))) + _, err = w.Write([]byte(fmt.Sprintf(`{"%s": %s}`, kind, b))) if err != nil { return err } diff --git a/pkg/pull.go b/pkg/pull.go new file mode 100644 index 00000000..30b7565a --- /dev/null +++ b/pkg/pull.go @@ -0,0 +1,150 @@ +package wedding + +import ( + "context" + "fmt" + "log" + "net/http" + "net/url" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func (s Service) pullImage(w http.ResponseWriter, r *http.Request) { + args := r.URL.Query() + + fromImage := args.Get("fromImage") + if fromImage == "" { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("image to pull is missing")) + return + } + + pullTag := args.Get("tag") + if pullTag == "" { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("tag to pull is missing")) + return + } + + if args.Get("repo") != "" { + w.WriteHeader(http.StatusNotImplemented) + w.Write([]byte("repo is not supported")) + return + } + + if args.Get("fromSrc") != "" { + w.WriteHeader(http.StatusNotImplemented) + w.Write([]byte("import from a file is not supported")) + return + } + + if args.Get("message") != "" { + w.WriteHeader(http.StatusNotImplemented) + w.Write([]byte("message is not supported")) + return + } + + if args.Get("platform") != "" { + w.WriteHeader(http.StatusNotImplemented) + w.Write([]byte("platform is not supported")) + return + } + + from := fmt.Sprintf("%s:%s", fromImage, pullTag) + to := fmt.Sprintf("wedding-registry:5000/images/%s", url.PathEscape(from)) + + dockerCfg, err := xRegistryAuth(r.Header.Get("X-Registry-Auth")).toDockerConfig() + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(fmt.Sprintf("extract registry config: %v", err))) + log.Printf("extract registry config: %v", err) + return + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "wedding-docker-config-", + }, + StringData: map[string]string{ + "config.json": dockerCfg.mustToJSON(), + }, + } + + secretClient := s.kubernetesClient.CoreV1().Secrets(s.namespace) + + secret, err = secretClient.Create(r.Context(), secret, metav1.CreateOptions{}) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + streamf(w, "Secret creation failed: %v\n", err) + log.Printf("create secret: %v", err) + return + } + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + err = secretClient.Delete(ctx, secret.Name, metav1.DeleteOptions{}) + if err != nil { + streamf(w, "Secret deletetion failed: %v\n", err) + log.Printf("delete secret: %v", err) + } + }() + + // TODO add timeout for script + buildScript := fmt.Sprintf(` +set -euo pipefail + +skopeo copy --dest-tls-verify=false docker://%s docker://%s +`, from, to) + + stream(w, buildScript) + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "wedding-pull-", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "skopeo", + Image: "mrliptontea/skopeo:1.2.0", + Command: []string{ + "sh", + "-c", + buildScript, + }, + VolumeMounts: []corev1.VolumeMount{ + { + MountPath: "/root/.docker", + Name: "docker-config", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "docker-config", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secret.Name, + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + } + + err = s.executePod(r.Context(), pod, w) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + stream(w, fmt.Sprintf("execute push: %v", err)) + log.Printf("execute push: %v", err) + return + } + + w.WriteHeader(http.StatusOK) +} diff --git a/pkg/push.go b/pkg/push.go index 2f9c0fd2..b60848d1 100644 --- a/pkg/push.go +++ b/pkg/push.go @@ -16,7 +16,7 @@ func (s Service) pushImage(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) args := r.URL.Query() - from := fmt.Sprintf("wedding-registry:5000/image/%s:%s", vars["name"], args.Get("tag")) + from := fmt.Sprintf("wedding-registry:5000/images/%s:%s", vars["name"], args.Get("tag")) to := fmt.Sprintf("%s:%s", vars["name"], args.Get("tag")) dockerCfg, err := xRegistryAuth(r.Header.Get("X-Registry-Auth")).toDockerConfig() diff --git a/pkg/registry_authentication.go b/pkg/registry_authentication.go index 8958a4f9..f582bcfa 100644 --- a/pkg/registry_authentication.go +++ b/pkg/registry_authentication.go @@ -40,8 +40,6 @@ func (x xRegistryConfig) toDockerConfig() (dockerConfig, error) { } creds := map[string]RegistryCred{} - log.Println(string(js)) - err = json.Unmarshal(js, &creds) if err != nil { return dockerConfig{}, fmt.Errorf("unmarshal registry authentications: %v", err) diff --git a/pkg/service.go b/pkg/service.go index e4e73d45..9dced00e 100644 --- a/pkg/service.go +++ b/pkg/service.go @@ -40,20 +40,21 @@ func (s *Service) routes() { router := mux.NewRouter() router.HandleFunc("/_ping", ping).Methods(http.MethodGet) router.HandleFunc("/session", ignored).Methods(http.MethodPost) - router.HandleFunc("/v"+apiVersion+"/version", version).Methods(http.MethodGet) + router.HandleFunc("/{apiVersion}/version", version).Methods(http.MethodGet) - router.HandleFunc("/v"+apiVersion+"/build", s.build).Methods(http.MethodPost) - router.HandleFunc("/v"+apiVersion+"/images/{name}/tag", s.tagImage).Methods(http.MethodPost) - router.HandleFunc("/v"+apiVersion+"/images/{name:.+}/push", s.pushImage).Methods(http.MethodPost) + router.HandleFunc("/{apiVersion}/build", s.build).Methods(http.MethodPost) + router.HandleFunc("/{apiVersion}/images/{name:.+}/tag", s.tagImage).Methods(http.MethodPost) + router.HandleFunc("/{apiVersion}/images/{name:.+}/push", s.pushImage).Methods(http.MethodPost) + router.HandleFunc("/{apiVersion}/images/create", s.pullImage).Methods(http.MethodPost) - router.HandleFunc("/v"+apiVersion+"/containers/prune", containersPrune).Methods(http.MethodPost) - router.HandleFunc("/v"+apiVersion+"/images/json", imagesJSON).Methods(http.MethodGet) - router.HandleFunc("/v"+apiVersion+"/build/prune", buildPrune).Methods(http.MethodPost) + router.HandleFunc("/{apiVersion}/containers/prune", containersPrune).Methods(http.MethodPost) + router.HandleFunc("/{apiVersion}/images/json", imagesJSON).Methods(http.MethodGet) + router.HandleFunc("/{apiVersion}/build/prune", buildPrune).Methods(http.MethodPost) router.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { stream(w, "This function is not supported by wedding.") w.WriteHeader(http.StatusNotImplemented) - log.Printf("NOT FOUND: %v", r) + log.Printf("501 - Not Implemented: %s %s", r.Method, r.URL) }) s.router = loggingMiddleware(router) @@ -62,7 +63,7 @@ func (s *Service) routes() { func loggingMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if !strings.HasPrefix(r.Header.Get("User-Agent"), "kube-probe/") { - log.Printf("%s %s\n", r.Method, r.URL) + log.Printf("HTTP Request %s %s\n", r.Method, r.URL) } next.ServeHTTP(w, r) diff --git a/pkg/tag.go b/pkg/tag.go index d6b06554..4d97bce7 100644 --- a/pkg/tag.go +++ b/pkg/tag.go @@ -4,7 +4,9 @@ import ( "fmt" "log" "net/http" - "os" + "net/url" + "regexp" + "strings" "github.com/gorilla/mux" corev1 "k8s.io/api/core/v1" @@ -16,9 +18,19 @@ func (s Service) tagImage(w http.ResponseWriter, r *http.Request) { args := r.URL.Query() from := fmt.Sprintf("wedding-registry:5000/digests@%s", vars["name"]) - to := fmt.Sprintf("wedding-registry:5000/image/%s:%s", args.Get("repo"), args.Get("tag")) + if !strings.HasPrefix(vars["name"], "sha256:") { + from = fmt.Sprintf("wedding-registry:5000/images/%s", url.PathEscape(escapePort(vars["name"]))) + } + + tag := args.Get("tag") + if tag == "" { + tag = "latest" + } - log.Printf("tag: %s as %s", from, to) + to := fmt.Sprintf( + "wedding-registry:5000/images/%s", + escapePort(fmt.Sprintf("%s:%s", args.Get("repo"), tag)), + ) // TODO add timeout for script buildScript := fmt.Sprintf(` @@ -47,13 +59,19 @@ skopeo copy --src-tls-verify=false --dest-tls-verify=false docker://%s docker:// }, } - err := s.executePod(r.Context(), pod, os.Stderr) + err := s.executePod(r.Context(), pod, w) if err != nil { w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fmt.Sprintf("execute tagging: %v", err))) + message(w, fmt.Sprintf("execute tagging: %v", err)) log.Printf("execute tagging: %v", err) return } w.WriteHeader(http.StatusCreated) } + +func escapePort(in string) string { + re := regexp.MustCompile(`:([0-9]+/)`) + escaped := re.ReplaceAll([]byte(in), []byte("_${1}")) + return string(escaped) +} diff --git a/services/wedding-registry.yaml b/services/wedding-registry.yaml index 4c8800fd..962997d1 100644 --- a/services/wedding-registry.yaml +++ b/services/wedding-registry.yaml @@ -49,6 +49,11 @@ spec: - sh - -c - "sleep 10" + env: + - name: REGISTRY_LOG_ACCESSLOG_DISABLED + value: "true" + - name: REGISTRY_LOG_LEVEL + value: "warn" volumeMounts: - name: data mountPath: /var/lib/registry diff --git a/tests/Dockerfile b/tests/Dockerfile new file mode 100644 index 00000000..47f7689a --- /dev/null +++ b/tests/Dockerfile @@ -0,0 +1,9 @@ +FROM tiltdev/tilt:v0.17.11 + +RUN apt update + +RUN apt install bats -y + +WORKDIR /tests +COPY tests /tests + diff --git a/tests/Tiltfile b/tests/Tiltfile new file mode 100644 index 00000000..9e985c16 --- /dev/null +++ b/tests/Tiltfile @@ -0,0 +1,15 @@ + +k8s_yaml('docker-build.yaml') +k8s_resource('test-docker-build', resource_deps=['wedding']) + +k8s_yaml('docker-pull-tag-push.yaml') +k8s_resource('test-docker-pull-tag-push', resource_deps=['wedding']) + +k8s_yaml('tilt-ci.yaml') +k8s_resource('test-tilt-ci', resource_deps=['wedding']) + +docker_build( + 'testing-image', + './..', + dockerfile='Dockerfile' +) diff --git a/tests/docker-build.yaml b/tests/docker-build.yaml new file mode 100644 index 00000000..1a8030e9 --- /dev/null +++ b/tests/docker-build.yaml @@ -0,0 +1,27 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: test-docker-build +spec: + backoffLimit: 1 + template: + metadata: + name: test-docker-build + spec: + containers: + - name: test + image: testing-image + command: + - bash + - -c + - | + set -euxo pipefail + + docker build ./docker + if docker build missing; then echo "this should fail"; false; else echo "exit code propagated"; fi + + echo "done" + env: + - name: "DOCKER_HOST" + value: "tcp://wedding:2376" + restartPolicy: Never diff --git a/tests/docker-pull-tag-push.yaml b/tests/docker-pull-tag-push.yaml new file mode 100644 index 00000000..bd3350bd --- /dev/null +++ b/tests/docker-pull-tag-push.yaml @@ -0,0 +1,33 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: test-docker-pull-tag-push +spec: + backoffLimit: 1 + template: + metadata: + name: test-docker-pull-tag-push + spec: + containers: + - name: test + image: testing-image + command: + - bash + - -c + - | + set -euxo pipefail + + docker pull alpine + if docker pull missing; then echo "this should fail"; false; else echo "exit code propagated"; fi + + docker tag alpine wedding-registry:5000/test-push:alpine + if docker tag missing b; then echo "this should fail"; false; else echo "exit code propagated"; fi + + docker push wedding-registry:5000/test-push:alpine + if docker push missing; then echo "this should fail"; false; else echo "exit code propagated"; fi + + echo "done" + env: + - name: "DOCKER_HOST" + value: "tcp://wedding:2376" + restartPolicy: Never diff --git a/use-case-1/Dockerfile b/tests/docker/Dockerfile similarity index 100% rename from use-case-1/Dockerfile rename to tests/docker/Dockerfile diff --git a/tests/tilt-ci.yaml b/tests/tilt-ci.yaml new file mode 100644 index 00000000..7a2a1a7b --- /dev/null +++ b/tests/tilt-ci.yaml @@ -0,0 +1,35 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: test-tilt-ci +spec: + backoffLimit: 1 + template: + metadata: + name: test-tilt-ci + spec: + serviceAccountName: wedding + containers: + - name: test + image: testing-image + command: + - bash + - -c + - | + set -euxo pipefail + + kubectl config set-cluster ci --server=https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT} --certificate-authority=/run/secrets/kubernetes.io/serviceaccount/ca.crt + kubectl config set-credentials ci --token=$(cat /run/secrets/kubernetes.io/serviceaccount/token) + kubectl config set-context ci --cluster=ci --user=ci --namespace=$(cat /run/secrets/kubernetes.io/serviceaccount/namespace) + kubectl config use-context ci + + cd tilt + + tilt ci + tilt down + + echo "done" + env: + - name: "DOCKER_HOST" + value: "tcp://wedding:2376" + restartPolicy: Never diff --git a/use-case-3/Dockerfile b/tests/tilt/Dockerfile similarity index 100% rename from use-case-3/Dockerfile rename to tests/tilt/Dockerfile diff --git a/use-case-3/Makefile b/tests/tilt/Makefile similarity index 100% rename from use-case-3/Makefile rename to tests/tilt/Makefile diff --git a/use-case-3/Tiltfile b/tests/tilt/Tiltfile similarity index 100% rename from use-case-3/Tiltfile rename to tests/tilt/Tiltfile diff --git a/use-case-3/kubernetes.yaml b/tests/tilt/kubernetes.yaml similarity index 100% rename from use-case-3/kubernetes.yaml rename to tests/tilt/kubernetes.yaml