Skip to content

Commit

Permalink
automaticly execute tests on code change
Browse files Browse the repository at this point in the history
  • Loading branch information
damoon committed Nov 13, 2020
1 parent fb17f10 commit a7b50b0
Show file tree
Hide file tree
Showing 19 changed files with 322 additions and 41 deletions.
21 changes: 2 additions & 19 deletions Tiltfile
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ load('ext://min_tilt_version', 'min_tilt_version')
min_tilt_version('0.15.0') # includes fix for auto_init+False with tilt ci

include('./services/Tiltfile')
include('./tests/Tiltfile')

k8s_yaml('deployment/kubernetes.yaml')

Expand Down Expand Up @@ -56,23 +57,5 @@ else:
k8s_resource(
'wedding',
port_forwards=['12376:2376'],
resource_deps=['setup-s3-bucket'],
)

local_resource ('docker build',
'DOCKER_HOST=tcp://127.0.0.1:12376 docker build ./use-case-1',
resource_deps=['wedding'],
)

local_resource ('docker pull tag push',
'DOCKER_HOST=tcp://127.0.0.1:12376 docker pull alpine && \
DOCKER_HOST=tcp://127.0.0.1:12376 docker tag alpine wedding-registry:5000/alpine-retag && \
DOCKER_HOST=tcp://127.0.0.1:12376 docker push wedding-registry:5000/alpine-retag',
resource_deps=['wedding'],
)

local_resource ('tilt ci',
'cd use-case-3 && DOCKER_HOST=tcp://127.0.0.1:12376 tilt ci --port 0 && \
tilt down',
resource_deps=['wedding'],
resource_deps=['setup-s3-bucket', 'wedding-registry'],
)
7 changes: 3 additions & 4 deletions pkg/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,6 @@ func buildParameters(r *http.Request) (*buildConfig, error) {
cfg.noCache = nocache == "1"

// registry authentitation
log.Printf("X-Registry-Config: %v", r.Header.Get("X-Registry-Config"))
dockerCfg, err := xRegistryConfig(r.Header.Get("X-Registry-Config")).toDockerConfig()
if err != nil {
return cfg, fmt.Errorf("extract registry config: %v", err)
Expand Down Expand Up @@ -255,7 +254,7 @@ func (o ObjectStore) presignContext(cfg *buildConfig) (string, error) {
Key: aws.String(cfg.contextFilePath),
})

url, err := objectRequest.Presign(time.Hour)
url, err := objectRequest.Presign(10 * time.Minute)
if err != nil {
return "", fmt.Errorf("presign GET %s: %v", cfg.contextFilePath, err)
}
Expand Down Expand Up @@ -321,7 +320,7 @@ func (s Service) executeBuild(ctx context.Context, cfg *buildConfig, w http.Resp

output := "--output type=image,push=true,name=wedding-registry:5000/digests"
if imageNames != "" {
output = fmt.Sprintf("--output type=image,push=true,name=\"%s\"", imageNames)
output = fmt.Sprintf("--output type=image,push=true,\"name=%s\"", imageNames)
}

// TODO add timeout for script
Expand Down Expand Up @@ -362,8 +361,8 @@ buildctl-daemonless.sh \
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "buildkit",
Image: "moby/buildkit:v0.7.2-rootless",
Name: "buildkit",
Command: []string{
"sh",
"-c",
Expand Down
10 changes: 9 additions & 1 deletion pkg/kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,13 +110,21 @@ func (s Service) podStatus(ctx context.Context, podName string) (corev1.PodPhase
return pod.Status.Phase, nil
}

func message(w io.Writer, message string) error {
return flush(w, "message", message)
}

func stream(w io.Writer, message string) error {
return flush(w, "stream", message)
}

func flush(w io.Writer, kind, message string) error {
b, err := json.Marshal(message)
if err != nil {
panic(err) // encode a string to json should not fail
}

_, err = w.Write([]byte(fmt.Sprintf(`{"stream": %s}`, b)))
_, err = w.Write([]byte(fmt.Sprintf(`{"%s": %s}`, kind, b)))
if err != nil {
return err
}
Expand Down
150 changes: 150 additions & 0 deletions pkg/pull.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
package wedding

import (
"context"
"fmt"
"log"
"net/http"
"net/url"
"time"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func (s Service) pullImage(w http.ResponseWriter, r *http.Request) {
args := r.URL.Query()

fromImage := args.Get("fromImage")
if fromImage == "" {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("image to pull is missing"))
return
}

pullTag := args.Get("tag")
if pullTag == "" {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("tag to pull is missing"))
return
}

if args.Get("repo") != "" {
w.WriteHeader(http.StatusNotImplemented)
w.Write([]byte("repo is not supported"))
return
}

if args.Get("fromSrc") != "" {
w.WriteHeader(http.StatusNotImplemented)
w.Write([]byte("import from a file is not supported"))
return
}

if args.Get("message") != "" {
w.WriteHeader(http.StatusNotImplemented)
w.Write([]byte("message is not supported"))
return
}

if args.Get("platform") != "" {
w.WriteHeader(http.StatusNotImplemented)
w.Write([]byte("platform is not supported"))
return
}

from := fmt.Sprintf("%s:%s", fromImage, pullTag)
to := fmt.Sprintf("wedding-registry:5000/images/%s", url.PathEscape(from))

dockerCfg, err := xRegistryAuth(r.Header.Get("X-Registry-Auth")).toDockerConfig()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(fmt.Sprintf("extract registry config: %v", err)))
log.Printf("extract registry config: %v", err)
return
}

secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "wedding-docker-config-",
},
StringData: map[string]string{
"config.json": dockerCfg.mustToJSON(),
},
}

secretClient := s.kubernetesClient.CoreV1().Secrets(s.namespace)

secret, err = secretClient.Create(r.Context(), secret, metav1.CreateOptions{})
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
streamf(w, "Secret creation failed: %v\n", err)
log.Printf("create secret: %v", err)
return
}
defer func() {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()

err = secretClient.Delete(ctx, secret.Name, metav1.DeleteOptions{})
if err != nil {
streamf(w, "Secret deletetion failed: %v\n", err)
log.Printf("delete secret: %v", err)
}
}()

// TODO add timeout for script
buildScript := fmt.Sprintf(`
set -euo pipefail
skopeo copy --dest-tls-verify=false docker://%s docker://%s
`, from, to)

stream(w, buildScript)

pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "wedding-pull-",
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "skopeo",
Image: "mrliptontea/skopeo:1.2.0",
Command: []string{
"sh",
"-c",
buildScript,
},
VolumeMounts: []corev1.VolumeMount{
{
MountPath: "/root/.docker",
Name: "docker-config",
},
},
},
},
Volumes: []corev1.Volume{
{
Name: "docker-config",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: secret.Name,
},
},
},
},
RestartPolicy: corev1.RestartPolicyNever,
},
}

err = s.executePod(r.Context(), pod, w)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
stream(w, fmt.Sprintf("execute push: %v", err))
log.Printf("execute push: %v", err)
return
}

w.WriteHeader(http.StatusOK)
}
2 changes: 1 addition & 1 deletion pkg/push.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ func (s Service) pushImage(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
args := r.URL.Query()

from := fmt.Sprintf("wedding-registry:5000/image/%s:%s", vars["name"], args.Get("tag"))
from := fmt.Sprintf("wedding-registry:5000/images/%s:%s", vars["name"], args.Get("tag"))
to := fmt.Sprintf("%s:%s", vars["name"], args.Get("tag"))

dockerCfg, err := xRegistryAuth(r.Header.Get("X-Registry-Auth")).toDockerConfig()
Expand Down
2 changes: 0 additions & 2 deletions pkg/registry_authentication.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ func (x xRegistryConfig) toDockerConfig() (dockerConfig, error) {
}
creds := map[string]RegistryCred{}

log.Println(string(js))

err = json.Unmarshal(js, &creds)
if err != nil {
return dockerConfig{}, fmt.Errorf("unmarshal registry authentications: %v", err)
Expand Down
19 changes: 10 additions & 9 deletions pkg/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,20 +40,21 @@ func (s *Service) routes() {
router := mux.NewRouter()
router.HandleFunc("/_ping", ping).Methods(http.MethodGet)
router.HandleFunc("/session", ignored).Methods(http.MethodPost)
router.HandleFunc("/v"+apiVersion+"/version", version).Methods(http.MethodGet)
router.HandleFunc("/{apiVersion}/version", version).Methods(http.MethodGet)

router.HandleFunc("/v"+apiVersion+"/build", s.build).Methods(http.MethodPost)
router.HandleFunc("/v"+apiVersion+"/images/{name}/tag", s.tagImage).Methods(http.MethodPost)
router.HandleFunc("/v"+apiVersion+"/images/{name:.+}/push", s.pushImage).Methods(http.MethodPost)
router.HandleFunc("/{apiVersion}/build", s.build).Methods(http.MethodPost)
router.HandleFunc("/{apiVersion}/images/{name:.+}/tag", s.tagImage).Methods(http.MethodPost)
router.HandleFunc("/{apiVersion}/images/{name:.+}/push", s.pushImage).Methods(http.MethodPost)
router.HandleFunc("/{apiVersion}/images/create", s.pullImage).Methods(http.MethodPost)

router.HandleFunc("/v"+apiVersion+"/containers/prune", containersPrune).Methods(http.MethodPost)
router.HandleFunc("/v"+apiVersion+"/images/json", imagesJSON).Methods(http.MethodGet)
router.HandleFunc("/v"+apiVersion+"/build/prune", buildPrune).Methods(http.MethodPost)
router.HandleFunc("/{apiVersion}/containers/prune", containersPrune).Methods(http.MethodPost)
router.HandleFunc("/{apiVersion}/images/json", imagesJSON).Methods(http.MethodGet)
router.HandleFunc("/{apiVersion}/build/prune", buildPrune).Methods(http.MethodPost)

router.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
stream(w, "This function is not supported by wedding.")
w.WriteHeader(http.StatusNotImplemented)
log.Printf("NOT FOUND: %v", r)
log.Printf("501 - Not Implemented: %s %s", r.Method, r.URL)
})

s.router = loggingMiddleware(router)
Expand All @@ -62,7 +63,7 @@ func (s *Service) routes() {
func loggingMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.HasPrefix(r.Header.Get("User-Agent"), "kube-probe/") {
log.Printf("%s %s\n", r.Method, r.URL)
log.Printf("HTTP Request %s %s\n", r.Method, r.URL)
}

next.ServeHTTP(w, r)
Expand Down
28 changes: 23 additions & 5 deletions pkg/tag.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ import (
"fmt"
"log"
"net/http"
"os"
"net/url"
"regexp"
"strings"

"github.com/gorilla/mux"
corev1 "k8s.io/api/core/v1"
Expand All @@ -16,9 +18,19 @@ func (s Service) tagImage(w http.ResponseWriter, r *http.Request) {
args := r.URL.Query()

from := fmt.Sprintf("wedding-registry:5000/digests@%s", vars["name"])
to := fmt.Sprintf("wedding-registry:5000/image/%s:%s", args.Get("repo"), args.Get("tag"))
if !strings.HasPrefix(vars["name"], "sha256:") {
from = fmt.Sprintf("wedding-registry:5000/images/%s", url.PathEscape(escapePort(vars["name"])))
}

tag := args.Get("tag")
if tag == "" {
tag = "latest"
}

log.Printf("tag: %s as %s", from, to)
to := fmt.Sprintf(
"wedding-registry:5000/images/%s",
escapePort(fmt.Sprintf("%s:%s", args.Get("repo"), tag)),
)

// TODO add timeout for script
buildScript := fmt.Sprintf(`
Expand Down Expand Up @@ -47,13 +59,19 @@ skopeo copy --src-tls-verify=false --dest-tls-verify=false docker://%s docker://
},
}

err := s.executePod(r.Context(), pod, os.Stderr)
err := s.executePod(r.Context(), pod, w)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(fmt.Sprintf("execute tagging: %v", err)))
message(w, fmt.Sprintf("execute tagging: %v", err))
log.Printf("execute tagging: %v", err)
return
}

w.WriteHeader(http.StatusCreated)
}

func escapePort(in string) string {
re := regexp.MustCompile(`:([0-9]+/)`)
escaped := re.ReplaceAll([]byte(in), []byte("_${1}"))
return string(escaped)
}
5 changes: 5 additions & 0 deletions services/wedding-registry.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,11 @@ spec:
- sh
- -c
- "sleep 10"
env:
- name: REGISTRY_LOG_ACCESSLOG_DISABLED
value: "true"
- name: REGISTRY_LOG_LEVEL
value: "warn"
volumeMounts:
- name: data
mountPath: /var/lib/registry
Expand Down
9 changes: 9 additions & 0 deletions tests/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
FROM tiltdev/tilt:v0.17.11

RUN apt update

RUN apt install bats -y

WORKDIR /tests
COPY tests /tests

15 changes: 15 additions & 0 deletions tests/Tiltfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@

k8s_yaml('docker-build.yaml')
k8s_resource('test-docker-build', resource_deps=['wedding'])

k8s_yaml('docker-pull-tag-push.yaml')
k8s_resource('test-docker-pull-tag-push', resource_deps=['wedding'])

k8s_yaml('tilt-ci.yaml')
k8s_resource('test-tilt-ci', resource_deps=['wedding'])

docker_build(
'testing-image',
'./..',
dockerfile='Dockerfile'
)
Loading

0 comments on commit a7b50b0

Please sign in to comment.