From 37815f79840c6a20abcfd5a0202b728c3f733e99 Mon Sep 17 00:00:00 2001 From: David Sauer Date: Mon, 10 Jan 2022 09:47:04 +0100 Subject: [PATCH] removed code for context dedup / dinner deployment --- Tiltfile | 41 ---- cmd/d8s/main.go | 213 +------------------- cmd/dinner/main.go | 208 ------------------- deployment/Dockerfile | 35 ---- deployment/entrypoint.sh | 2 - deployment/kubernetes.yaml | 58 ------ pkg/build.go | 91 --------- pkg/chunks.go | 124 ------------ pkg/mock.go | 68 ------- pkg/service.go | 87 -------- service-dependencies/Tiltfile | 39 ---- service-dependencies/dind.yaml | 75 ------- service-dependencies/docker-hub-mirror.yaml | 99 --------- service-dependencies/minio.yaml | 128 ------------ service-dependencies/registry.yaml | 104 ---------- tests/2gi-build/Dockerfile | 5 - tests/2gi-build/eat-2gi.go | 8 - tests/Tiltfile | 26 +-- tests/build.sh | 18 ++ tests/d8s-build.sh | 5 - tests/d8s-build/Dockerfile | 3 - tests/d8s-build/index.html | 1 - tests/docker-build.sh | 11 - tests/docker-inspect.sh | 10 - tests/docker-max-memory.sh | 9 - tests/docker-pull-tag-push.sh | 16 -- tests/inspect.sh | 10 + tests/pull-tag.sh | 20 ++ 28 files changed, 61 insertions(+), 1453 deletions(-) delete mode 100644 cmd/dinner/main.go delete mode 100644 deployment/Dockerfile delete mode 100755 deployment/entrypoint.sh delete mode 100644 deployment/kubernetes.yaml delete mode 100644 pkg/build.go delete mode 100644 pkg/chunks.go delete mode 100644 pkg/mock.go delete mode 100644 pkg/service.go delete mode 100644 service-dependencies/Tiltfile delete mode 100644 service-dependencies/dind.yaml delete mode 100644 service-dependencies/docker-hub-mirror.yaml delete mode 100644 service-dependencies/minio.yaml delete mode 100644 service-dependencies/registry.yaml delete mode 100644 tests/2gi-build/Dockerfile delete mode 100644 tests/2gi-build/eat-2gi.go create mode 100644 tests/build.sh delete mode 100644 tests/d8s-build.sh delete mode 100644 tests/d8s-build/Dockerfile delete mode 100644 tests/d8s-build/index.html delete mode 100644 tests/docker-build.sh delete mode 100644 tests/docker-inspect.sh delete mode 100644 tests/docker-max-memory.sh delete mode 100644 tests/docker-pull-tag-push.sh create mode 100644 tests/inspect.sh create mode 100644 tests/pull-tag.sh diff --git a/Tiltfile b/Tiltfile index 7e636f2a..c882a4ea 100644 --- a/Tiltfile +++ b/Tiltfile @@ -2,45 +2,4 @@ disable_snapshots() analytics_settings(enable=False) allow_k8s_contexts(os.getenv("TILT_ALLOW_CONTEXT")) -include('./service-dependencies/Tiltfile') include('./tests/Tiltfile') - -k8s_yaml('deployment/kubernetes.yaml') - -target='prod' -live_update=[] -if os.environ.get('PROD', '') == '': - target='build-env' - live_update=[ - sync('pkg', '/app/pkg'), - sync('cmd', '/app/cmd'), - sync('go.mod', '/app/go.mod'), - sync('go.sum', '/app/go.sum'), - run('go install -v ./cmd/dinner'), - ] - -docker_build( - 'davedamoon/dinner:latest', - '.', - dockerfile='deployment/Dockerfile', - target=target, - build_args={"SOURCE_BRANCH":"development", "SOURCE_COMMIT":"development"}, - only=[ 'go.mod' - , 'go.sum' - , 'pkg' - , 'cmd' - , 'deployment' - ], - ignore=[ '.git' - , '*/*_test.go' - , 'deployment/kubernetes.yaml' - ], - live_update=live_update, -) - -k8s_resource( - 'dinner', - port_forwards=['12375:2375'], - resource_deps=['minio-buckets', 'dind'], - labels=["application"], -) diff --git a/cmd/d8s/main.go b/cmd/d8s/main.go index 57433733..2490f3c0 100644 --- a/cmd/d8s/main.go +++ b/cmd/d8s/main.go @@ -2,16 +2,11 @@ package main import ( "bufio" - "bytes" "context" - "crypto/sha256" - "encoding/hex" "fmt" "io" "log" - "net" "net/http" - "net/http/httputil" "net/url" "os" "os/exec" @@ -41,9 +36,8 @@ const ( ) var ( - gitHash string - gitRef = "latest" - uploadBottlenecks = MutexMap{} + gitHash string + gitRef = "latest" ) type MutexMap struct { @@ -71,8 +65,8 @@ func main() { Usage: "The client for dinner.", Commands: []*cli.Command{ { - Name: "run", - Usage: "Connect to dinner server and set DOCKER_HOST for started process.", + Name: "up", + Usage: "Connect to docker in docker and set DOCKER_HOST for started process.", Flags: []cli.Flag{ &cli.BoolFlag{ Name: "verbose", @@ -97,7 +91,7 @@ func main() { EnvVars: []string{"D8S_NAMESPACE"}, }, }, - Action: run, + Action: up, }, { Name: "version", @@ -129,7 +123,7 @@ func version(c *cli.Context) error { return nil } -func run(c *cli.Context) error { +func up(c *cli.Context) error { args := c.Args() if args.First() == "" { return fmt.Errorf("command missing") @@ -153,12 +147,7 @@ func run(c *cli.Context) error { localAddr, stopCh := portForward(pod, config, verbose) defer close(stopCh) - localPort, err := localServer("http://"+localAddr, verbose) - if err != nil { - return fmt.Errorf("parse local address: %v", err) - } - - err = executeCommand(c.Args(), fmt.Sprintf("127.0.0.1:%d", localPort)) + err = executeCommand(c.Args(), localAddr) if err != nil { return fmt.Errorf("command failed with %s", err) } @@ -168,6 +157,10 @@ func run(c *cli.Context) error { // https://stackoverflow.com/questions/50435564/use-kubectl-context-in-kubernetes-client-go func setupKubernetesClient(kubeconfig, context, namespace string) (*kubernetes.Clientset, *rest.Config, string, string, error) { + // TODO: verify kubernetes context is ok + // https://github.com/tilt-dev/tilt/blob/fe386b5cc967383972bf73f8cbe6514c604100f8/internal/k8s/env.go#L38 + // https://github.com/turbine-kreuzberg/dind-nurse/blob/main/Tiltfile#L3 + configLoader := clientcmd.NewDefaultClientConfigLoadingRules() configLoader.ExplicitPath = kubeconfig @@ -378,187 +371,3 @@ func executeCommand(args cli.Args, localAddr string) error { return nil } - -func localServer(localAddr string, verbose bool) (int, error) { - targetURL, err := url.Parse(localAddr) - if err != nil { - return 0, err - } - - proxy := httputil.NewSingleHostReverseProxy(targetURL) - - mux := http.NewServeMux() - mux.HandleFunc("/", uploadContextHandlerFunc(proxy, localAddr, verbose)) - - listener, err := net.Listen("tcp", ":0") - if err != nil { - return 0, err - } - - go http.Serve(listener, mux) - - return listener.Addr().(*net.TCPAddr).Port, nil -} - -func uploadContextHandlerFunc(proxy *httputil.ReverseProxy, localAddr string, verbose bool) http.HandlerFunc { - re := regexp.MustCompile(`^/[^/]+/build$`) - - return func(w http.ResponseWriter, r *http.Request) { - if !re.MatchString(r.URL.Path) { - proxy.ServeHTTP(w, r) - return - } - - proxy.ServeHTTP(w, r) - return - - chunker := chunker.New(r.Body, staticPol) - chunksList := &bytes.Buffer{} - - for { - c, err := chunker.Next(nil) - - if err == io.EOF { - break - } - - if err != nil { - log.Printf("searching next chunk: %v", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - ok := verifyChunk(w, r, chunksList, c, localAddr, verbose) - if !ok { - return - } - } - - r.Body = io.NopCloser(bytes.NewReader(chunksList.Bytes())) - r.Header.Add("d8s-chunked", "true") - - proxy.ServeHTTP(w, r) - } -} - -func verifyChunk(w http.ResponseWriter, r *http.Request, chunksList *bytes.Buffer, c chunker.Chunk, localAddr string, verbose bool) bool { - hash, err := hashData(bytes.NewBuffer(c.Data)) - - // lock on hash to avoid race condition and double uploads - unlock := uploadBottlenecks.Lock(hash) - defer unlock() - - chunksList.Write(hash) - - found, err := chunkExists(r.Context(), localAddr+"/_chunks", hash) - if err != nil { - log.Printf("chunk #%d deduplication: %v", c.Cut, err) - w.WriteHeader(http.StatusInternalServerError) - return false - } - - if found { - if verbose { - log.Printf("SKIP uploading %d bytes", c.Length) - } - return true - } - - if verbose { - log.Printf("uploading %d bytes", c.Length) - } - - err = uploadChunk(r.Context(), localAddr+"/_chunks", bytes.NewBuffer(c.Data)) - if err != nil { - log.Printf("uploading chunk #%d: %v", c.Cut, err) - w.WriteHeader(http.StatusInternalServerError) - return false - } - - return true -} - -func chunkExists(ctx context.Context, target string, hash []byte) (bool, error) { - client := &http.Client{ - Transport: &http.Transport{ - Dial: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).Dial, - TLSHandshakeTimeout: 10 * time.Second, - ResponseHeaderTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - }, - } - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) - if err != nil { - return false, fmt.Errorf("create request for chunk deduplication: %v", err) - } - - hashHex := make([]byte, hex.EncodedLen(len(hash))) - hex.Encode(hashHex, hash) - - q := req.URL.Query() - q.Add("hash", string(hashHex)) - req.URL.RawQuery = q.Encode() - - resp, err := client.Do(req) - if err != nil { - return false, fmt.Errorf("chunk deduplication request: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode == http.StatusOK { - return true, nil - } - - if resp.StatusCode == http.StatusNotFound { - return false, nil - } - - return false, fmt.Errorf("chunk deduplication request returned status code %d", resp.StatusCode) - -} - -func hashData(r io.Reader) ([]byte, error) { - h := sha256.New() - - _, err := io.Copy(h, r) - if err != nil { - return []byte{}, err - } - - return h.Sum(nil), nil -} - -func uploadChunk(ctx context.Context, target string, data io.Reader) error { - client := &http.Client{ - Transport: &http.Transport{ - Dial: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).Dial, - TLSHandshakeTimeout: 10 * time.Second, - ResponseHeaderTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - }, - } - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, data) - if err != nil { - return fmt.Errorf("create request for chunk upload: %v", err) - } - - resp, err := client.Do(req) - if err != nil { - return fmt.Errorf("chunk upload: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("chunk upload returned status code %d", resp.StatusCode) - } - - return nil -} diff --git a/cmd/dinner/main.go b/cmd/dinner/main.go deleted file mode 100644 index 23d40cb2..00000000 --- a/cmd/dinner/main.go +++ /dev/null @@ -1,208 +0,0 @@ -package main - -import ( - "context" - "fmt" - "io/ioutil" - "log" - "net/http" - "net/http/httputil" - "net/url" - "os" - "os/signal" - "strings" - "syscall" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" - dinner "github.com/damoon/d8s/pkg" - "github.com/urfave/cli/v2" -) - -var ( - gitHash string - gitRef string -) - -func main() { - app := &cli.App{ - Name: "Dinner", - Usage: "Proxy to minimize data transfer volume of docker build contexts.", - Action: run, - Commands: []*cli.Command{ - { - Name: "server", - Usage: "Start the server.", - Flags: []cli.Flag{ - &cli.StringFlag{Name: "addr", Value: ":2375", Usage: "Address to run service on."}, - &cli.StringFlag{Name: "s3-endpoint", Required: true, Usage: "s3 endpoint."}, - &cli.StringFlag{Name: "s3-access-key-file", Required: true, Usage: "Path to s3 access key."}, - &cli.StringFlag{Name: "s3-secret-key-file", Required: true, Usage: "Path to s3 secret access key."}, - &cli.BoolFlag{Name: "s3-ssl", Value: true, Usage: "s3 uses SSL."}, - &cli.StringFlag{Name: "s3-location", Value: "us-east-1", Usage: "s3 bucket location."}, - &cli.StringFlag{Name: "s3-bucket", Required: true, Usage: "s3 bucket name."}, - &cli.StringFlag{Name: "upstream", Required: true, Usage: "Upstream to forward requests to."}, - }, - Action: run, - }, - { - Name: "version", - Usage: "Show the version", - Action: func(c *cli.Context) error { - _, err := os.Stdout.WriteString(fmt.Sprintf("version: %s\ngit commit: %s", gitRef, gitHash)) - if err != nil { - return err - } - - return nil - }, - }, - }, - } - - err := app.Run(os.Args) - if err != nil { - log.Println(err) - os.Exit(1) - } -} - -func run(c *cli.Context) error { - log.Printf("version: %v", gitRef) - log.Printf("git commit: %v", gitHash) - - log.Println("set up storage") - - storage, err := setupObjectStore( - c.String("s3-endpoint"), - c.String("s3-access-key-file"), - c.String("s3-secret-key-file"), - c.Bool("s3-ssl"), - c.String("s3-location"), - c.String("s3-bucket")) - if err != nil { - return fmt.Errorf("setup minio s3 client: %v", err) - } - - log.Println("set up upstream proxy") - upstream, err := setupUpstreamProxy(c.String("upstream")) - if err != nil { - return fmt.Errorf("setup upstream proxy: %v", err) - } - - log.Println("set up service") - - svc := dinner.NewService(gitHash, gitRef, storage, upstream) - - svcServer := httpServer(svc, c.String("addr")) - - log.Println("starting server") - - go mustListenAndServe(svcServer) - - log.Println("running") - - awaitShutdown() - - ctx, cancel := context.WithTimeout(context.Background(), dinner.MaxExecutionTime) - defer cancel() - - err = shutdown(ctx, svcServer) - if err != nil { - return fmt.Errorf("shutdown service server: %v", err) - } - - log.Println("shutdown complete") - - return nil -} - -func setupObjectStore( - endpoint, accessKeyPath, secretKeyPath string, - useSSL bool, - region, bucket string, -) (*dinner.ObjectStore, error) { - accessKeyBytes, err := ioutil.ReadFile(accessKeyPath) - if err != nil { - return nil, fmt.Errorf("reading secret access key from %s: %v", accessKeyPath, err) - } - - secretKeyBytes, err := ioutil.ReadFile(secretKeyPath) - if err != nil { - return nil, fmt.Errorf("reading secret access key from %s: %v", secretKeyPath, err) - } - - accessKey := strings.TrimSpace(string(accessKeyBytes)) - secretKey := strings.TrimSpace(string(secretKeyBytes)) - - endpointProtocol := "http" - if useSSL { - endpointProtocol = "https" - } - - s3Config := &aws.Config{ - Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""), - Endpoint: aws.String(fmt.Sprintf("%s://%s", endpointProtocol, endpoint)), - Region: aws.String(region), - DisableSSL: aws.Bool(!useSSL), - S3ForcePathStyle: aws.Bool(true), - } - - sess, err := session.NewSession(s3Config) - if err != nil { - return nil, fmt.Errorf("set up aws session: %v", err) - } - - s3Client := s3.New(sess) - - return &dinner.ObjectStore{ - Client: s3Client, - Uploader: s3manager.NewUploader(sess), - Bucket: bucket, - }, nil -} - -func setupUpstreamProxy(upstream string) (*httputil.ReverseProxy, error) { - upstreamURL, err := url.Parse(upstream) - if err != nil { - return nil, err - } - - return httputil.NewSingleHostReverseProxy(upstreamURL), nil -} - -func httpServer(h http.Handler, addr string) *http.Server { - httpServer := &http.Server{ - ReadTimeout: dinner.MaxExecutionTime, - WriteTimeout: dinner.MaxExecutionTime, - } - httpServer.Addr = addr - httpServer.Handler = h - - return httpServer -} - -func mustListenAndServe(srv *http.Server) { - err := srv.ListenAndServe() - if err != nil && err != http.ErrServerClosed { - log.Fatal(err) - } -} - -func awaitShutdown() { - stop := make(chan os.Signal, 2) - signal.Notify(stop, os.Interrupt, syscall.SIGTERM) - <-stop -} - -func shutdown(ctx context.Context, srv *http.Server) error { - err := srv.Shutdown(ctx) - if err != nil { - return err - } - - return nil -} diff --git a/deployment/Dockerfile b/deployment/Dockerfile deleted file mode 100644 index 9d74305e..00000000 --- a/deployment/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -# build-env ################################################### -FROM golang:1.16.4-buster AS build-env - -WORKDIR /app - -RUN apt-get update -RUN apt-get install -y entr - -COPY deployment/entrypoint.sh /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] - -COPY go.mod . -COPY go.sum . -RUN go mod download - -ARG SOURCE_BRANCH -ARG SOURCE_COMMIT - -ENV CGO_ENABLED=0 -ENV GOOS=linux - -COPY cmd cmd -COPY pkg pkg -RUN go install -v \ - -ldflags="-X main.gitRef=${SOURCE_BRANCH} -X main.gitHash=${SOURCE_COMMIT}" \ - -installsuffix cgo \ - ./cmd/dinner - -############################################################### -FROM alpine:3.13.5 AS prod - -RUN apk add --no-cache ca-certificates -WORKDIR /root/ -COPY --from=build-env /go/bin/dinner /usr/local/bin/dinner -ENTRYPOINT [ "dinner", "server" ] diff --git a/deployment/entrypoint.sh b/deployment/entrypoint.sh deleted file mode 100755 index 17c1fd8c..00000000 --- a/deployment/entrypoint.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -find /go/bin/dinner | entr -d -r dinner server $@ diff --git a/deployment/kubernetes.yaml b/deployment/kubernetes.yaml deleted file mode 100644 index c2cf739c..00000000 --- a/deployment/kubernetes.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: dinner -spec: - selector: - app: dinner - ports: - - name: dinner - port: 2375 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dinner - labels: - app: dinner -spec: - selector: - matchLabels: - app: dinner - template: - metadata: - labels: - app: dinner - spec: - containers: - - name: dinner - image: davedamoon/dinner:latest - args: - - --s3-endpoint=d8s-minio:9000 - - --s3-access-key-file=/secret/minio/MINIO_ACCESS_KEY - - --s3-secret-key-file=/secret/minio/MINIO_SECRET_KEY - - --s3-ssl=false - - --s3-bucket=chunks - - --upstream=http://d8s-dind:2375 - ports: - - name: dinner - containerPort: 2375 - readinessProbe: - httpGet: - path: /_ping - port: 2375 - volumeMounts: - - name: minio - mountPath: "/secret/minio" - readOnly: true - resources: - requests: - memory: "1Gi" # increased to allow go to compile, and tilt to hot reload - cpu: "100m" - limits: - memory: "1Gi" - cpu: "200m" - volumes: - - name: minio - secret: - secretName: d8s-minio diff --git a/pkg/build.go b/pkg/build.go deleted file mode 100644 index 2eb626b1..00000000 --- a/pkg/build.go +++ /dev/null @@ -1,91 +0,0 @@ -package d8s - -import ( - "bytes" - "context" - "fmt" - "io" - "io/ioutil" - "log" - "net/http" - "os" - - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" -) - -// ObjectStore manages access to a S3 compatible file store. -type ObjectStore struct { - Client *s3.S3 - Uploader *s3manager.Uploader - Bucket string -} - -func (s Service) build(w http.ResponseWriter, r *http.Request) { - chunked := r.Header.Get("d8s-chunked") != "" - - if !chunked { - s.upstream.ServeHTTP(w, r) - return - } - - ctx := r.Context() - - tempfile, err := ioutil.TempFile("", "build-context-restore") - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fmt.Sprintf("restore context: %v", err))) - log.Printf("restore context: %v", err) - return - } - defer os.Remove(tempfile.Name()) - - buf := new(bytes.Buffer) - _, err = buf.ReadFrom(r.Body) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fmt.Sprintf("read chunk list: %v", err))) - log.Printf("read chunk list: %v", err) - return - } - - err = s.restoreContext(ctx, buf, tempfile) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fmt.Sprintf("restore context: %v", err))) - log.Printf("restore context: %v", err) - return - } - - _, err = tempfile.Seek(0, io.SeekStart) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fmt.Sprintf("seek restored context file: %v", err))) - log.Printf("seek restored context file: %v", err) - return - } - - r.Body = tempfile - s.upstream.ServeHTTP(w, r) -} - -func (s Service) restoreContext(ctx context.Context, chunkList io.Reader, w io.Writer) error { - hash := make([]byte, 32) - - for { - _, err := chunkList.Read(hash) - if err == io.EOF { - return nil - } - - chunk, err := s.restoreChunk(ctx, hash) - if err != nil { - return err - } - - _, err = io.Copy(w, chunk) - if err != nil { - return err - } - } -} diff --git a/pkg/chunks.go b/pkg/chunks.go deleted file mode 100644 index 16eb80ff..00000000 --- a/pkg/chunks.go +++ /dev/null @@ -1,124 +0,0 @@ -package d8s - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "log" - "net/http" - "path/filepath" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" -) - -// ChunkExists returns a 200 OK in case the chunk-hash is known or a 404 Not Found in case the chunk-hash is unknown. -func (s Service) chunkExists(w http.ResponseWriter, r *http.Request) { - hash := r.URL.Query().Get("hash") - - found, err := s.objectStore.chunkExists(r.Context(), hash) - if err != nil { - log.Printf("look up chunk \"%s\": %v", hash, err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - if !found { - w.WriteHeader(http.StatusNotFound) - return - } - - w.WriteHeader(http.StatusOK) -} - -func (o ObjectStore) chunkExists(ctx context.Context, chunkName string) (bool, error) { - path := filepath.Join("chunks", chunkName) - - _, err := o.Client.HeadObjectWithContext(ctx, &s3.HeadObjectInput{ - Bucket: aws.String(o.Bucket), - Key: aws.String(path), - }) - - if err == nil { - return true, nil - } - - aerr, ok := err.(awserr.Error) - if !ok { - return false, fmt.Errorf("failed to cast error to awserr.Error") - } - - switch aerr.Code() { - case s3.ErrCodeNoSuchBucket: - return false, fmt.Errorf("bucket %s does not exist: %v", o.Bucket, err) - case s3.ErrCodeNoSuchKey: - return false, nil - case "NotFound": - return false, nil - } - - return false, err -} - -// AddChunk stores a chunk for later reuse. -func (s Service) addChunk(w http.ResponseWriter, r *http.Request) { - buf := bytes.NewBuffer(make([]byte, 0)) - reader := io.TeeReader(r.Body, buf) - - hash, err := hashData(reader) - if err != nil { - log.Printf("calculate chunk hash: %v", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - hashHex := make([]byte, hex.EncodedLen(len(hash))) - hex.Encode(hashHex, hash) - - path := filepath.Join("chunks", string(hashHex)) - - _, err = s.objectStore.Uploader.UploadWithContext(r.Context(), &s3manager.UploadInput{ - Bucket: aws.String(s.objectStore.Bucket), - Key: aws.String(path), - ContentType: aws.String("application/octet-stream"), - Body: buf, - }) - if err != nil { - log.Printf("upload chunk to object store: %v", err) - w.WriteHeader(http.StatusInternalServerError) - } -} - -func hashData(r io.Reader) ([]byte, error) { - h := sha256.New() - - _, err := io.Copy(h, r) - if err != nil { - return []byte{}, err - } - - return h.Sum(nil), nil -} - -// RestoreChunk stores a chunk for later reuse. -func (s Service) restoreChunk(ctx context.Context, hash []byte) (io.Reader, error) { - hashHex := make([]byte, hex.EncodedLen(len(hash))) - hex.Encode(hashHex, hash) - - path := filepath.Join("chunks", string(hashHex)) - - object, err := s.objectStore.Client.GetObjectWithContext(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.objectStore.Bucket), - Key: aws.String(path), - }) - if err != nil { - return &bytes.Buffer{}, err - } - - return object.Body, nil -} diff --git a/pkg/mock.go b/pkg/mock.go deleted file mode 100644 index b77c7fd4..00000000 --- a/pkg/mock.go +++ /dev/null @@ -1,68 +0,0 @@ -package d8s - -import ( - "encoding/json" - "fmt" - "io" - "net/http" -) - -type output struct { - w io.Writer -} - -func (o output) Write(b []byte) (int, error) { - i := len(b) - - b, err := json.Marshal(string(b)) - if err != nil { - return 0, err - } - - msg := fmt.Sprintf(`{"stream": %s}`, b) - - _, err = o.w.Write([]byte(msg)) - if err != nil { - return 0, err - } - - if f, ok := o.w.(http.Flusher); ok { - f.Flush() - } else { - return 0, fmt.Errorf("stream can not be flushed") - } - - return i, nil -} - -func (o output) Errorf(e string, args ...interface{}) error { - return o.Error(fmt.Sprintf(e, args...)) -} - -func (o output) Error(e string) error { - b, err := json.Marshal(string(e)) - if err != nil { - return err - } - - msg := fmt.Sprintf(`{"error": %s, "errorDetail": {"code": %d, "message": %s}}`, b, 1, b) - - _, err = o.w.Write([]byte(msg)) - if err != nil { - return err - } - - if f, ok := o.w.(http.Flusher); ok { - f.Flush() - } else { - return fmt.Errorf("stream can not be flushed") - } - - return nil -} - -func ping(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Api-Version", apiVersion) - w.Header().Set("Docker-Experimental", "false") - w.WriteHeader(http.StatusOK) -} diff --git a/pkg/service.go b/pkg/service.go deleted file mode 100644 index 98d6eb52..00000000 --- a/pkg/service.go +++ /dev/null @@ -1,87 +0,0 @@ -package d8s - -import ( - "net/http" - "net/http/httputil" - "strings" - "time" - - "github.com/gorilla/mux" - "golang.org/x/sync/semaphore" -) - -const ( - // MaxExecutionTime is the longest time allowed for a command to run. - MaxExecutionTime = 30 * time.Minute - - apiVersion = "1.40" - buildkitImage = "moby/buildkit:v0.9.3-rootless" - skopeoImage = "ghcr.io/utopia-planitia/skopeo-image@sha256:130836bd82e5f3a856f659e22f0e9d97c545ff0d955807b806595ec4874d5f37" - buildMemory = "2147483648" // 2Gi default - buildCPUQuota = 100_000 // results in 1 cpu - buildCPUPeriod = 100_000 // 100ms is the default of docker - skopeoMemory = "100Mi" - skopeoCPU = "200m" -) - -var ( - semBuild = semaphore.NewWeighted(1) - semSkopeo = semaphore.NewWeighted(5) -) - -// Service runs the dinner server. -type Service struct { - router http.Handler - objectStore *ObjectStore - upstream *httputil.ReverseProxy -} - -// NewService creates a new service server and initiates the routes. -func NewService(gitHash, gitRef string, objectStore *ObjectStore, upstream *httputil.ReverseProxy) *Service { - srv := &Service{ - objectStore: objectStore, - upstream: upstream, - } - - srv.routes(gitHash, gitRef) - - return srv -} - -func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { - s.router.ServeHTTP(w, r) -} - -func (s *Service) routes(gitHash, gitRef string) { - router := mux.NewRouter() - router.HandleFunc("/_ping", ping).Methods(http.MethodGet) - - router.HandleFunc("/{apiVersion}/build", s.build).Methods(http.MethodPost) - - router.HandleFunc("/_chunks", s.chunkExists).Methods(http.MethodGet) - router.HandleFunc("/_chunks", s.addChunk).Methods(http.MethodPost) - - router.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - s.upstream.ServeHTTP(w, r) - }) - - s.router = loggingMiddleware(router) -} - -func loggingMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !strings.HasPrefix(r.Header.Get("User-Agent"), "kube-probe/") { - // log.Printf("HTTP Request %s %s\n", r.Method, r.URL) - } - - next.ServeHTTP(w, r) - }) -} - -func ignored(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) -} - -func missing(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotFound) -} diff --git a/service-dependencies/Tiltfile b/service-dependencies/Tiltfile deleted file mode 100644 index d6172790..00000000 --- a/service-dependencies/Tiltfile +++ /dev/null @@ -1,39 +0,0 @@ -k8s_yaml('minio.yaml') -k8s_resource( - 'd8s-minio', - new_name='minio', - port_forwards=9000, - labels=["dependencies"], -) -k8s_resource( - 'setup-minio-buckets', - new_name='minio-buckets', - resource_deps=['minio'], - labels=["dependencies"], -) - -k8s_yaml('dind.yaml') -k8s_resource( - 'd8s-dind', - new_name='dind', - port_forwards=['22375:2375'], - labels=["dependencies"], -) - -k8s_yaml('registry.yaml') -k8s_resource( - 'd8s-registry', - new_name='registry', - port_forwards=5000, - resource_deps=['minio-buckets'], - labels=["dependencies"], -) - -k8s_yaml('docker-hub-mirror.yaml') -k8s_resource( - 'd8s-docker-hub-mirror', - new_name='docker-hub-mirror', - port_forwards=5001, - resource_deps=['minio-buckets'], - labels=["dependencies"], -) diff --git a/service-dependencies/dind.yaml b/service-dependencies/dind.yaml deleted file mode 100644 index 38b1ce74..00000000 --- a/service-dependencies/dind.yaml +++ /dev/null @@ -1,75 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: d8s-dind -spec: - ports: - - name: dind - port: 2375 - selector: - app: d8s-dind ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: d8s-dind -spec: - selector: - matchLabels: - app: d8s-dind - strategy: - type: Recreate - replicas: 1 - template: - metadata: - labels: - app: d8s-dind - spec: - containers: - - name: dind - image: docker:20.10.11-dind - args: - - --tls=false - - --insecure-registry=d8s-registry:5000 - - --mtu=1300 - - --registry-mirror=http://d8s-docker-hub-mirror:5000 - - --insecure-registry=d8s-docker-hub-mirror:5000 - env: - - name: DOCKER_TLS_CERTDIR - value: "" - - name: DOCKER_BUILDKIT - value: "1" - securityContext: - privileged: true - ports: - - name: http - containerPort: 2375 - resources: - limits: - cpu: "1000m" - memory: "8Gi" - requests: - cpu: "100m" - memory: "8Gi" - readinessProbe: - httpGet: - path: /_ping - port: 2375 - volumeMounts: - - name: data - mountPath: /var/lib/docker - volumes: - - name: data - persistentVolumeClaim: - claimName: d8s-dind ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: d8s-dind -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 100Gi diff --git a/service-dependencies/docker-hub-mirror.yaml b/service-dependencies/docker-hub-mirror.yaml deleted file mode 100644 index 713c9196..00000000 --- a/service-dependencies/docker-hub-mirror.yaml +++ /dev/null @@ -1,99 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: d8s-docker-hub-mirror -spec: - ports: - - name: http - port: 5000 - selector: - app: d8s-docker-hub-mirror ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: d8s-docker-hub-mirror -spec: - selector: - matchLabels: - app: d8s-docker-hub-mirror - replicas: 1 - template: - metadata: - labels: - app: d8s-docker-hub-mirror - spec: - containers: - - name: d8s-docker-hub-mirror - image: registry:2.7.1 - ports: - - name: http - containerPort: 5000 - resources: - limits: - cpu: 1000m - memory: "500Mi" - requests: - cpu: "100m" - memory: "500Mi" - livenessProbe: - httpGet: - path: /v2/ - port: 5000 - readinessProbe: - httpGet: - path: /v2/_catalog?n=1 - port: 5000 - env: - - name: REGISTRY_LOG_ACCESSLOG_DISABLED - value: "true" - - name: REGISTRY_LOG_LEVEL - value: "warn" - - name: REGISTRY_STORAGE_S3_ACCESSKEY - valueFrom: - secretKeyRef: - name: d8s-minio - key: MINIO_ACCESS_KEY - - name: REGISTRY_STORAGE_S3_SECRETKEY - valueFrom: - secretKeyRef: - name: d8s-minio - key: MINIO_SECRET_KEY - - name: REGISTRY_HTTP_SECRET - value: abc123 - volumeMounts: - - name: config - mountPath: /etc/docker/registry/ - volumes: - - name: config - configMap: - name: d8s-docker-hub-mirror ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: d8s-docker-hub-mirror -data: - config.yml: | - version: 0.1 - log: - fields: - service: registry-mirror - storage: - s3: - region: us-east-1 - regionendpoint: http://d8s-minio:9000 - bucket: docker-hub-mirror - encrypt: false - secure: true - v4auth: true - chunksize: 5242880 - rootdirectory: / - redirect: - disable: true - proxy: - remoteurl: https://registry-1.docker.io - http: - addr: :5000 - headers: - X-Content-Type-Options: [nosniff] diff --git a/service-dependencies/minio.yaml b/service-dependencies/minio.yaml deleted file mode 100644 index d250c424..00000000 --- a/service-dependencies/minio.yaml +++ /dev/null @@ -1,128 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: d8s-minio -spec: - ports: - - name: minio - port: 9000 - selector: - app: d8s-minio ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: d8s-minio -spec: - selector: - matchLabels: - app: d8s-minio - strategy: - type: Recreate - replicas: 1 - template: - metadata: - labels: - app: d8s-minio - spec: - containers: - - name: minio - image: minio/minio:RELEASE.2020-12-29T23-29-29Z - args: - - server - - /home/shared - envFrom: - - secretRef: - name: d8s-minio - ports: - - name: http - containerPort: 9000 - resources: - limits: - cpu: "1000m" - memory: "1Gi" - requests: - cpu: "100m" - memory: "1Gi" - livenessProbe: - httpGet: - path: /minio/health/live - port: 9000 - readinessProbe: - httpGet: - path: /minio/health/cluster - port: 9000 - lifecycle: - preStop: - exec: - command: - - sh - - -c - - until [[ $(curl --silent --fail --max-time 5 --write-out "%{http_code}" http://127.0.0.1:9000/minio/health/cluster\?maintenance\=true) = "200" ]]; do sleep 1; done - volumeMounts: - - name: data - mountPath: /home/shared - volumes: - - name: data - persistentVolumeClaim: - claimName: d8s-minio ---- -apiVersion: v1 -kind: Secret -metadata: - name: d8s-minio -type: Opaque -stringData: - MINIO_ACCESS_KEY: minio123 - MINIO_SECRET_KEY: minio456 ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: d8s-minio -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 20Gi ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: setup-minio-buckets -spec: - backoffLimit: 20 - template: - metadata: - name: setup-minio-bucket - spec: - containers: - - name: mc - image: minio/mc:RELEASE.2020-12-18T10-53-53Z - command: - - sh - - -c - - | - set -euxo pipefail - mc config host add minio http://d8s-minio:9000 $(cat /secret/minio/MINIO_ACCESS_KEY) $(cat /secret/minio/MINIO_SECRET_KEY) - until timeout 10 mc admin info minio; do sleep 1; done - mc mb minio/chunks --ignore-existing - mc mb minio/registry --ignore-existing - mc mb minio/docker-hub-mirror --ignore-existing - resources: - limits: - cpu: 20m - memory: 100Mi - requests: - cpu: 20m - memory: 100Mi - volumeMounts: - - name: minio - mountPath: "/secret/minio" - readOnly: true - volumes: - - name: minio - secret: - secretName: d8s-minio - restartPolicy: OnFailure diff --git a/service-dependencies/registry.yaml b/service-dependencies/registry.yaml deleted file mode 100644 index 644cdb3d..00000000 --- a/service-dependencies/registry.yaml +++ /dev/null @@ -1,104 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: d8s-registry -spec: - ports: - - name: http - port: 5000 - selector: - app: d8s-registry ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: d8s-registry -spec: - selector: - matchLabels: - app: d8s-registry - replicas: 1 - template: - metadata: - labels: - app: d8s-registry - spec: - containers: - - name: d8s-registry - image: registry:2.7.1 - ports: - - name: http - containerPort: 5000 - resources: - limits: - cpu: 1000m - memory: "500Mi" - requests: - cpu: "100m" - memory: "500Mi" - livenessProbe: - httpGet: - path: /v2/ - port: 5000 - readinessProbe: - httpGet: - path: /v2/_catalog?n=1 - port: 5000 - env: - - name: REGISTRY_LOG_ACCESSLOG_DISABLED - value: "true" - - name: REGISTRY_LOG_LEVEL - value: "warn" - - name: REGISTRY_STORAGE_S3_ACCESSKEY - valueFrom: - secretKeyRef: - name: d8s-minio - key: MINIO_ACCESS_KEY - - name: REGISTRY_STORAGE_S3_SECRETKEY - valueFrom: - secretKeyRef: - name: d8s-minio - key: MINIO_SECRET_KEY - - name: REGISTRY_HTTP_SECRET - value: abc123 - volumeMounts: - - name: config - mountPath: /etc/docker/registry/ - volumes: - - name: config - configMap: - name: d8s-registry ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: d8s-registry -data: - config.yml: | - version: 0.1 - log: - fields: - service: registry - storage: - s3: - region: us-east-1 - regionendpoint: http://d8s-minio:9000 - bucket: registry - encrypt: false - secure: true - v4auth: true - chunksize: 5242880 - rootdirectory: / - delete: - enabled: true - maintenance: - uploadpurging: - enabled: false - readonly: - enabled: false - redirect: - disable: true - http: - addr: :5000 - headers: - X-Content-Type-Options: [nosniff] diff --git a/tests/2gi-build/Dockerfile b/tests/2gi-build/Dockerfile deleted file mode 100644 index fa34a538..00000000 --- a/tests/2gi-build/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM golang:1.15.5-buster - -COPY eat-2gi.go . - -RUN go run eat-2gi.go diff --git a/tests/2gi-build/eat-2gi.go b/tests/2gi-build/eat-2gi.go deleted file mode 100644 index fd427ded..00000000 --- a/tests/2gi-build/eat-2gi.go +++ /dev/null @@ -1,8 +0,0 @@ -package main - -func main() { - var data = [1968 * 1024 * 1024]byte{} - for i := 0; i < len(data); i++ { - data[i] = 1 - } -} diff --git a/tests/Tiltfile b/tests/Tiltfile index 8e4d1c13..9c096c6c 100644 --- a/tests/Tiltfile +++ b/tests/Tiltfile @@ -1,40 +1,18 @@ local_resource ('test build', - 'timeout 120 bash docker-build.sh', + 'timeout 120 bash build.sh', deps=['..'], - resource_deps=['dinner'], - allow_parallel=True, labels=["tests"], ) -local_resource ('test pull tag push', +local_resource ('test pull tag', 'timeout 200 bash docker-pull-tag-push.sh', deps=['..'], - resource_deps=['dinner'], - allow_parallel=True, labels=["tests"], ) local_resource ('test inspect', 'timeout 200 bash docker-inspect.sh', deps=['..'], - resource_deps=['dinner'], - allow_parallel=True, - labels=["tests"], -) - -local_resource ('test max memory', - 'timeout 120 bash docker-max-memory.sh', - deps=['..'], - resource_deps=['dinner'], - allow_parallel=True, - labels=["tests"], -) - -local_resource ('test d8s build', - 'timeout 120 bash d8s-build.sh', - deps=['..'], - resource_deps=['dinner'], - allow_parallel=True, labels=["tests"], ) diff --git a/tests/build.sh b/tests/build.sh new file mode 100644 index 00000000..487ab8d7 --- /dev/null +++ b/tests/build.sh @@ -0,0 +1,18 @@ +#!bash +set -uexo pipefail + +go run ../cmd/d8s run \ + docker build \ + -t build-test-a \ + -t build-test-b \ + -f ./docker/dir/Dockerfile \ + ./docker + +if go run ../cmd/d8s run docker build ./docker-broken; then + echo "this should fail"; + false; +else + echo "exit code propagated"; +fi + +echo "done" diff --git a/tests/d8s-build.sh b/tests/d8s-build.sh deleted file mode 100644 index 8bf47dc0..00000000 --- a/tests/d8s-build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!bash -set -uexo pipefail - -go run ../cmd/d8s run \ - docker build -t d8s-build-test ./d8s-build -f ./d8s-build/Dockerfile diff --git a/tests/d8s-build/Dockerfile b/tests/d8s-build/Dockerfile deleted file mode 100644 index ced5607b..00000000 --- a/tests/d8s-build/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM nginx - -COPY index.html /usr/share/nginx/html/index.html diff --git a/tests/d8s-build/index.html b/tests/d8s-build/index.html deleted file mode 100644 index 78981922..00000000 --- a/tests/d8s-build/index.html +++ /dev/null @@ -1 +0,0 @@ -a diff --git a/tests/docker-build.sh b/tests/docker-build.sh deleted file mode 100644 index 5cb34aaa..00000000 --- a/tests/docker-build.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!bash -set -uexo pipefail -export DOCKER_HOST=tcp://127.0.0.1:12375 -export DOCKER_BUILDKIT=0 -until docker version; do sleep 1; done - -docker build -t d8s-build-test-a -t d8s-build-test-b ./docker -f ./docker/dir/Dockerfile - -if docker build ./docker-broken; then echo "this should fail"; false; else echo "exit code propagated"; fi - -echo "done" diff --git a/tests/docker-inspect.sh b/tests/docker-inspect.sh deleted file mode 100644 index 34b6027c..00000000 --- a/tests/docker-inspect.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!bash -set -uexo pipefail -export DOCKER_HOST=tcp://127.0.0.1:12375 -export DOCKER_BUILDKIT=0 -until docker version; do sleep 1; done - -docker pull alpine -docker inspect alpine - -echo "done" diff --git a/tests/docker-max-memory.sh b/tests/docker-max-memory.sh deleted file mode 100644 index e9b5189c..00000000 --- a/tests/docker-max-memory.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!bash -set -uexo pipefail -export DOCKER_HOST=tcp://127.0.0.1:12375 -export DOCKER_BUILDKIT=0 -until docker version; do sleep 1; done - -docker build ./2gi-build - -echo "done" diff --git a/tests/docker-pull-tag-push.sh b/tests/docker-pull-tag-push.sh deleted file mode 100644 index 0b1cae76..00000000 --- a/tests/docker-pull-tag-push.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!bash -set -uexo pipefail -export DOCKER_HOST=tcp://127.0.0.1:12375 -export DOCKER_BUILDKIT=0 -until docker version; do sleep 1; done - -docker pull alpine -if docker pull missing; then echo "this should fail"; false; else echo "exit code propagated"; fi - -docker tag alpine d8s-registry:5000/test-push:alpine -if docker tag missing b; then echo "this should fail"; false; else echo "exit code propagated"; fi - -docker push d8s-registry:5000/test-push:alpine -if docker push missing; then echo "this should fail"; false; else echo "exit code propagated"; fi - -echo "done" diff --git a/tests/inspect.sh b/tests/inspect.sh new file mode 100644 index 00000000..0355f2a5 --- /dev/null +++ b/tests/inspect.sh @@ -0,0 +1,10 @@ +#!bash +set -uexo pipefail + +go run ../cmd/d8s run \ + docker pull alpine + +go run ../cmd/d8s run \ + docker inspect alpine + +echo "done" diff --git a/tests/pull-tag.sh b/tests/pull-tag.sh new file mode 100644 index 00000000..8af6f86b --- /dev/null +++ b/tests/pull-tag.sh @@ -0,0 +1,20 @@ +#!bash +set -uexo pipefail + +docker pull alpine +if docker pull missing; then + echo "this should fail"; + false; +else + echo "exit code propagated"; +fi + +docker tag alpine d8s-registry:5000/test-push:alpine +if docker tag missing b; then + echo "this should fail"; + false; +else + echo "exit code propagated"; +fi + +echo "done"