From 50cb6f5664cb6f0ceabe9fd3098a3114948f617a Mon Sep 17 00:00:00 2001 From: David Sauer Date: Mon, 24 May 2021 15:54:10 +0200 Subject: [PATCH] added endpoint to store chucks of build contexts --- go.sum | 3 ++ pkg/build.go | 47 +++++++++------------ pkg/chunks.go | 108 +++++++++++++++++++++++++++++++++++++++++++++++++ pkg/service.go | 3 ++ 4 files changed, 134 insertions(+), 27 deletions(-) create mode 100644 pkg/chunks.go diff --git a/go.sum b/go.sum index f8687c80..baac8bf6 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,7 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -125,6 +126,7 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -147,6 +149,7 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= diff --git a/pkg/build.go b/pkg/build.go index 2704259a..c4acc047 100644 --- a/pkg/build.go +++ b/pkg/build.go @@ -54,7 +54,24 @@ func (s Service) build(w http.ResponseWriter, r *http.Request) { return } - s.buildInKubernetes(w, r, cfg) + ctx := r.Context() + + err = s.objectStore.storeContext(ctx, r.Body, cfg) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(fmt.Sprintf("store context: %v", err))) + log.Printf("execute build: %v", err) + return + } + defer func() { + s.objectStore.deleteContext(ctx, cfg) + }() + + err = s.executeBuild(ctx, cfg, w) + if err != nil { + log.Printf("execute build: %v", err) + return + } } func buildParameters(r *http.Request) (*buildConfig, error) { @@ -182,32 +199,11 @@ func printBuildHelpText(w http.ResponseWriter, err error) { } } -func (s Service) buildInKubernetes(w http.ResponseWriter, r *http.Request, cfg *buildConfig) { - ctx := r.Context() - - err := s.objectStore.storeContext(ctx, r.Body, cfg) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fmt.Sprintf("store context: %v", err))) - log.Printf("execute build: %v", err) - return - } - defer func() { - s.objectStore.deleteContext(ctx, cfg) - }() - - err = s.executeBuild(ctx, cfg, w) - if err != nil { - log.Printf("execute build: %v", err) - return - } -} - func (o ObjectStore) storeContext(ctx context.Context, r io.Reader, cfg *buildConfig) error { path := fmt.Sprintf("%d.tar", time.Now().UnixNano()) cfg.contextFilePath = path - _, err := o.Uploader.Upload(&s3manager.UploadInput{ + _, err := o.Uploader.UploadWithContext(ctx, &s3manager.UploadInput{ Bucket: aws.String(o.Bucket), Key: aws.String(path), ContentType: aws.String("application/x-tar"), @@ -316,10 +312,7 @@ set -euo pipefail unset x echo download build context -cd ~ -touch ~/context -rm -rf ~/context -mkdir ~/context && cd ~/context +cd ~ && mkdir context && cd context wget -O - "%s" | tar -xf - set -x diff --git a/pkg/chunks.go b/pkg/chunks.go new file mode 100644 index 00000000..c07c881f --- /dev/null +++ b/pkg/chunks.go @@ -0,0 +1,108 @@ +package wedding + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "log" + "net/http" + "path/filepath" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" +) + +// ChunkExists returns a 200 OK in case the chunk-hash is known or a 404 Not Found in case the chunk-hash is unknown. +func (s Service) chunkExists(w http.ResponseWriter, r *http.Request) { + hash := r.URL.Query().Get("hash") + + found, err := s.objectStore.chunkExists(r.Context(), hash) + if err != nil { + log.Printf("look up chunk \"%s\": %v", hash, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + if !found { + w.WriteHeader(http.StatusNotFound) + return + } + + w.WriteHeader(http.StatusOK) +} + +func (o ObjectStore) chunkExists(ctx context.Context, chunkName string) (bool, error) { + path := filepath.Join("chunks", chunkName) + + _, err := o.Client.HeadObjectWithContext(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(o.Bucket), + Key: aws.String(path), + }) + + if err == nil { + return true, nil + } + + aerr, ok := err.(awserr.Error) + if !ok { + return false, fmt.Errorf("failed to cast error to awserr.Error") + } + + switch aerr.Code() { + case s3.ErrCodeNoSuchBucket: + return false, fmt.Errorf("bucket %s does not exist: %v", o.Bucket, err) + case s3.ErrCodeNoSuchKey: + return false, nil + case "NotFound": + return false, nil + } + + return false, err +} + +// AddChunk stores a chunk for later reuse. +func (s Service) addChunk(w http.ResponseWriter, r *http.Request) { + buf := bytes.NewBuffer(make([]byte, 0)) + reader := io.TeeReader(r.Body, buf) + + hash, err := hashData(reader) + if err != nil { + log.Printf("calculate chunk hash: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + hashHex := make([]byte, hex.EncodedLen(len(hash))) + hex.Encode(hashHex, hash) + + log.Printf("calculated hash: %v", string(hashHex)) + + path := filepath.Join("chunks", string(hashHex)) + + _, err = s.objectStore.Uploader.UploadWithContext(r.Context(), &s3manager.UploadInput{ + Bucket: aws.String(s.objectStore.Bucket), + Key: aws.String(path), + ContentType: aws.String("application/octet-stream"), + Body: buf, + }) + if err != nil { + log.Printf("upload chunk to object store: %v", err) + w.WriteHeader(http.StatusInternalServerError) + } +} + +func hashData(r io.Reader) ([]byte, error) { + h := sha256.New() + + _, err := io.Copy(h, r) + if err != nil { + return []byte{}, err + } + + return h.Sum(nil), nil +} diff --git a/pkg/service.go b/pkg/service.go index 3cf1bec7..0ba95e0a 100644 --- a/pkg/service.go +++ b/pkg/service.go @@ -73,6 +73,9 @@ func (s *Service) routes(gitHash, gitRef string) { router.HandleFunc("/{apiVersion}/images/json", imagesJSON).Methods(http.MethodGet) router.HandleFunc("/{apiVersion}/build/prune", buildPrune).Methods(http.MethodPost) + router.HandleFunc("/_chunks", s.chunkExists).Methods(http.MethodGet) + router.HandleFunc("/_chunks", s.addChunk).Methods(http.MethodPost) + router.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotImplemented) w.Write([]byte("This function is not supported by wedding."))