From 77988b77ac18181d9c2bb5618a439a2eeb452629 Mon Sep 17 00:00:00 2001 From: Reaper Gelera Date: Tue, 30 Apr 2024 20:45:42 +0530 Subject: [PATCH 1/6] move minio to a seperate file --- go.mod | 3 +++ go.sum | 10 +++++++ storage/minio.go | 63 +++++++++++++++++++++++++++++++++++++++++++ storage/storage.go | 66 +++------------------------------------------- 4 files changed, 80 insertions(+), 62 deletions(-) create mode 100644 storage/minio.go diff --git a/go.mod b/go.mod index f63a46a..709509a 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.19 require ( github.com/Masterminds/semver v1.5.0 + github.com/aws/aws-sdk-go v1.51.31 + github.com/barelyhuman/go v0.2.2 github.com/google/go-github/v53 v53.2.0 github.com/joho/godotenv v1.5.1 github.com/minio/minio-go/v7 v7.0.66 @@ -18,6 +20,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.5 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect diff --git a/go.sum b/go.sum index 0ebca8a..5584cc5 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,10 @@ github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3Q github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/aws/aws-sdk-go v1.51.31 h1:4TM+sNc+Dzs7wY1sJ0+J8i60c6rkgnKP1pvPx8ghsSY= +github.com/aws/aws-sdk-go v1.51.31/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/barelyhuman/go v0.2.2 h1:Lpk1XrlP40F3II8BibVzViZUOJ1GgDdzXUBb8ENwb0U= +github.com/barelyhuman/go v0.2.2/go.mod h1:hox2iDYZAarjpS7jKQeYIi2F+qMA8KLMtCws++L2sSY= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= @@ -25,6 +29,10 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -129,6 +137,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/storage/minio.go b/storage/minio.go new file mode 100644 index 0000000..de73a4b --- /dev/null +++ b/storage/minio.go @@ -0,0 +1,63 @@ +package storage + +import ( + "context" + "fmt" + "net/url" + "os" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type MinioStorage struct { + Client *minio.Client + BucketName string +} + +func (s *MinioStorage) Connect() error { + endpoint := os.Getenv("MINIO_URL") + useSSL := false + + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4( + os.Getenv("MINIO_ROOT_USER"), + os.Getenv("MINIO_ROOT_PASSWORD"), + ""), + Secure: useSSL, + }) + if err != nil { + return err + } + + s.Client = client + return nil +} + +func (s *MinioStorage) Upload(objectName string, filePath string) error { + + ctx := context.Background() + bucketName := s.BucketName + + info, err := s.Client.FPutObject(ctx, bucketName, objectName, filePath, minio.PutObjectOptions{}) + if err != nil { + return err + } + + fmt.Println(info) + return nil +} + +func (s *MinioStorage) GetSignedURL(objectName string, alias string) (string, error) { + reqParams := make(url.Values) + reqParams.Set("response-content-disposition", "attachment; filename=\""+alias+"\"") + presignedURL, err := s.Client.PresignedGetObject(context.Background(), s.BucketName, objectName, time.Minute*5, reqParams) + if err != nil { + return "", err + } + + url := os.Getenv("MINIO_URL_PREFIX") + presignedURL.RequestURI() + + return url, nil +} diff --git a/storage/storage.go b/storage/storage.go index 4a33a9d..f88a847 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -1,65 +1,7 @@ package storage -import ( - "context" - "fmt" - "net/url" - "os" - "time" - - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" -) - -type Storage struct { - Client *minio.Client - BucketName string -} - -func (s *Storage) Connect() error { - endpoint := os.Getenv("MINIO_URL") - useSSL := false - - // Initialize minio client object. - client, err := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4( - os.Getenv("MINIO_ROOT_USER"), - os.Getenv("MINIO_ROOT_PASSWORD"), - ""), - Secure: useSSL, - }) - if err != nil { - return err - } - - s.Client = client - return nil -} - -func (s *Storage) Upload(objectName string, filePath string) error { - - ctx := context.Background() - bucketName := s.BucketName - - // Upload the zip file with FPutObject - info, err := s.Client.FPutObject(ctx, bucketName, objectName, filePath, minio.PutObjectOptions{}) - if err != nil { - return err - } - - fmt.Println(info) - return nil -} - -func (s *Storage) GetSignedURL(objectName string, alias string) (string, error) { - reqParams := make(url.Values) - reqParams.Set("response-content-disposition", "attachment; filename=\""+alias+"\"") - presignedURL, err := s.Client.PresignedGetObject(context.Background(), s.BucketName, objectName, time.Minute*5, reqParams) - if err != nil { - return "", err - } - - url := os.Getenv("MINIO_URL_PREFIX") + presignedURL.RequestURI() - - return url, nil +type Storage interface { + Connect() error + Upload(objectName, filePath string) error + GetSignedURL(objectName, path string) error } From 98470285ff3d1e8087e7263cf37e037f626e7cf6 Mon Sep 17 00:00:00 2001 From: Reaper Gelera Date: Wed, 1 May 2024 16:55:19 +0530 Subject: [PATCH 2/6] feat: rudimentary impl of from cache --- .docker-compose-with-storage.yml | 68 ---------------------- cmd/goblin-api/main.go | 92 +++++++++++++++++------------- docker-compose.yml | 25 +++++++- go.mod | 18 +----- go.sum | 46 ++------------- storage/minio.go | 98 ++++++++++++++++++++------------ storage/storage.go | 7 ++- 7 files changed, 152 insertions(+), 202 deletions(-) delete mode 100644 .docker-compose-with-storage.yml diff --git a/.docker-compose-with-storage.yml b/.docker-compose-with-storage.yml deleted file mode 100644 index cde9b85..0000000 --- a/.docker-compose-with-storage.yml +++ /dev/null @@ -1,68 +0,0 @@ -# FIXME: -# DISABLED STORAGE FOR THE INITIAL VERSION SINCE BUILDING IS LIMITED TO A FEW SECONDS - -version: "3.4" -services: - api: - build: "." - container_name: goblin_api - depends_on: - - createbuckets - env_file: .env - expose: - - "3000:3000" - environment: - MINIO_URL: goblin_minio:9000 - restart: always - ports: - - "3000:3000" - volumes: - - "./:/usr/src/app" - - createbuckets: - depends_on: - minio: - condition: service_healthy - entrypoint: > - /bin/sh -c " - /usr/bin/mc config host add myminio http://goblin_minio:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD; - /usr/bin/mc mb -p myminio/$$BUCKET_NAME; - /usr/bin/mc policy set public myminio/$$BUCKET_NAME; - exit 0; - " - env_file: - - .env - image: minio/mc - - minio: - command: 'server --console-address ":9001" /home/files' - container_name: goblin_minio - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 30s - timeout: 20s - retries: 3 - env_file: - - .env - image: "minio/minio:RELEASE.2022-01-08T03-11-54Z" - expose: - - "9000:9000" - - "9001:9001" - volumes: - - "goblin:/home/files" - - nginx: - container_name: goblin_nginx - depends_on: - # - minio - - api - hostname: nginx - image: "nginx:1.19.2-alpine" - ports: - - "80:80" - - "9000:9000" - volumes: - - "./nginx.conf:/etc/nginx/nginx.conf:ro" - -volumes: - goblin: ~ diff --git a/cmd/goblin-api/main.go b/cmd/goblin-api/main.go index d051917..57b7d2b 100644 --- a/cmd/goblin-api/main.go +++ b/cmd/goblin-api/main.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "errors" "flag" "fmt" @@ -12,16 +13,16 @@ import ( "path/filepath" "strings" + "github.com/barelyhuman/go/env" "github.com/barelyhuman/goblin/build" "github.com/barelyhuman/goblin/resolver" + "github.com/barelyhuman/goblin/storage" "github.com/joho/godotenv" ) var shTemplates *template.Template var serverURL string - -// FIXME: Disabled storage and caching for initial version -// var storageClient *storage.Storage +var storageClient storage.Storage func HandleRequest(rw http.ResponseWriter, req *http.Request) { path := req.URL.Path @@ -40,6 +41,7 @@ func HandleRequest(rw http.ResponseWriter, req *http.Request) { } if strings.HasPrefix(path, "/binary") { + log.Print("handle binary") fetchBinary(rw, req) return } @@ -63,20 +65,12 @@ func StartServer(port string) { } } -func envDefault(key string, def string) string { - if s := os.Getenv(key); len(strings.TrimSpace(s)) == 0 { - return def - } else { - return s - } -} - // TODO: cleanup code // TODO: move everything into their own interface/structs func main() { envFile := flag.String("env", ".env", "path to read the env config from") - portFlag := envDefault("PORT", "3000") + portFlag := env.Get("PORT", "3000") flag.Parse() @@ -88,19 +82,28 @@ func main() { } shTemplates = template.Must(template.ParseGlob("templates/*")) - serverURL = envDefault("ORIGIN_URL", "http://localhost:3000") + serverURL = env.Get("ORIGIN_URL", "http://localhost:"+portFlag) - // FIXME: Disabled storage and caching for initial version - // storageClient = &storage.Storage{} - // storageClient.BucketName = os.Getenv("BUCKET_NAME") - // err := storageClient.Connect() - // if err != nil { - // log.Fatal(err) - // } + if isStorageEnabled() { + storageClient = storage.NewAWSStorage(env.Get("STORAGE_BUCKET", "goblin-cache")) + err := storageClient.Connect() + if err != nil { + log.Fatal(err) + } + } StartServer(portFlag) } +func isStorageEnabled() bool { + useStorageEnv := env.Get("STORAGE_ENABLED", "false") + useStorage := false + if useStorageEnv == "true" { + useStorage = true + } + return useStorage +} + func normalizePackage(pkg string) string { // strip leading protocol pkg = strings.Replace(pkg, "https://", "", 1) @@ -228,16 +231,19 @@ func fetchBinary(rw http.ResponseWriter, req *http.Request) { Module: mod, } - // TODO: check the storage for existing binary for the module - // and return from the storage instead - immutable(rw) - // FIXME: Disabled storage and caching for initial version - // var buf bytes.Buffer - // err := bin.WriteBuild(io.MultiWriter(rw, &buf)) + artifactName := constructArtifactName(bin) - err := bin.WriteBuild(io.MultiWriter(rw)) + if isStorageEnabled() && storageClient.HasObject(artifactName) { + url, _ := storageClient.GetSignedURL(artifactName) + log.Println("From cache") + http.Redirect(rw, req, url, http.StatusSeeOther) + return + } + + var buf bytes.Buffer + err := bin.WriteBuild(io.MultiWriter(rw, &buf)) if err != nil { rw.WriteHeader(http.StatusInternalServerError) @@ -245,21 +251,29 @@ func fetchBinary(rw http.ResponseWriter, req *http.Request) { return } + if isStorageEnabled() { + err = storageClient.Upload( + artifactName, + buf, + ) + + if err != nil { + log.Println("Failed to upload", err) + } + } + err = bin.Cleanup() if err != nil { log.Println("cleaning binary build", err) } +} - // FIXME: Disabled storage and caching for initial version - // err = storageClient.Upload(bin.Module, bin.Dest) - // if err != nil { - // fmt.Fprint(rw, err.Error()) - // return - // } - - // url, err := storageClient.GetSignedURL(bin.Module, bin.Name) - // if err != nil { - // fmt.Fprint(rw, err.Error()) - // return - // } +func constructArtifactName(bin *build.Binary) string { + var artifactName strings.Builder + artifactName.Write([]byte(bin.Name)) + artifactName.Write([]byte("-")) + artifactName.Write([]byte(bin.OS)) + artifactName.Write([]byte("-")) + artifactName.Write([]byte(bin.Arch)) + return artifactName.String() } diff --git a/docker-compose.yml b/docker-compose.yml index a19f373..6e1775e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,25 @@ services: - "3000:3000" volumes: - "./:/usr/src/app" + profiles: [app] + + minio: + image: "minio/minio:RELEASE.2024-05-01T01-11-10Z" + container_name: goblin_minio + command: 'server --console-address ":9001" /home/files' + healthcheck: + test: ["CMD", "curl", "-I", "http://127.0.0.1:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + env_file: + - .env + expose: + - "9000:9000" + - "9001:9001" + volumes: + - "goblin:/data" + profiles: [storage] nginx: container_name: goblin_nginx @@ -27,4 +46,8 @@ services: - "80:80" - "9000:9000" volumes: - - "./nginx.conf:/etc/nginx/nginx.conf:ro" \ No newline at end of file + - "./nginx.conf:/etc/nginx/nginx.conf:ro" + profiles: [app] + +volumes: + goblin: \ No newline at end of file diff --git a/go.mod b/go.mod index 709509a..13c093c 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,10 @@ go 1.19 require ( github.com/Masterminds/semver v1.5.0 - github.com/aws/aws-sdk-go v1.51.31 github.com/barelyhuman/go v0.2.2 github.com/google/go-github/v53 v53.2.0 github.com/joho/godotenv v1.5.1 - github.com/minio/minio-go/v7 v7.0.66 + github.com/minio/minio-go v6.0.14+incompatible github.com/tj/go-semver v1.0.0 golang.org/x/oauth2 v0.16.0 ) @@ -16,20 +15,10 @@ require ( require ( github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/cloudflare/circl v1.3.7 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.5 // indirect - github.com/klauspost/cpuid/v2 v2.2.6 // indirect - github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/sha256-simd v1.0.1 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/rs/xid v1.5.0 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/stretchr/testify v1.8.1 // indirect golang.org/x/crypto v0.18.0 // indirect golang.org/x/net v0.20.0 // indirect @@ -37,5 +26,4 @@ require ( golang.org/x/text v0.14.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.32.0 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 5584cc5..7d444e0 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3Q github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/aws/aws-sdk-go v1.51.31 h1:4TM+sNc+Dzs7wY1sJ0+J8i60c6rkgnKP1pvPx8ghsSY= -github.com/aws/aws-sdk-go v1.51.31/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/barelyhuman/go v0.2.2 h1:Lpk1XrlP40F3II8BibVzViZUOJ1GgDdzXUBb8ENwb0U= github.com/barelyhuman/go v0.2.2/go.mod h1:hox2iDYZAarjpS7jKQeYIi2F+qMA8KLMtCws++L2sSY= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= @@ -13,8 +11,8 @@ github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBS github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= @@ -26,45 +24,18 @@ github.com/google/go-github/v53 v53.2.0 h1:wvz3FyF53v4BK+AsnvCmeNhf8AkTaeh2SoYu/ github.com/google/go-github/v53 v53.2.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSXGt7mOsAWEloao= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.5 h1:d4vBd+7CHydUqpFBgUEKkSdtSugf9YFmSkvUYPquI5E= -github.com/klauspost/compress v1.17.5/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= -github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw= -github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs= -github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= -github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= +github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= @@ -99,7 +70,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -134,11 +104,7 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/storage/minio.go b/storage/minio.go index de73a4b..2869f01 100644 --- a/storage/minio.go +++ b/storage/minio.go @@ -1,63 +1,87 @@ package storage import ( - "context" + "bytes" "fmt" + "log" "net/url" - "os" "time" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/barelyhuman/go/env" + "github.com/minio/minio-go" ) -type MinioStorage struct { - Client *minio.Client - BucketName string +type S3Storage struct { + client *minio.Client + bucket string } -func (s *MinioStorage) Connect() error { - endpoint := os.Getenv("MINIO_URL") - useSSL := false - - client, err := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4( - os.Getenv("MINIO_ROOT_USER"), - os.Getenv("MINIO_ROOT_PASSWORD"), - ""), - Secure: useSSL, - }) +func NewAWSStorage(bucket string) *S3Storage { + clientId := env.Get("STORAGE_CLIENT_ID", "") + clientSecret := env.Get("STORAGE_CLIENT_SECRET", "") + endpoint := env.Get("STORAGE_ENDPOINT", "") + ssl := true + + // Initiate a client using DigitalOcean Spaces. + client, err := minio.New(endpoint, clientId, clientSecret, ssl) if err != nil { - return err + log.Fatal(err) } - s.Client = client - return nil -} - -func (s *MinioStorage) Upload(objectName string, filePath string) error { + bucketExists, _ := client.BucketExists(bucket) - ctx := context.Background() - bucketName := s.BucketName + if !bucketExists { + err := client.MakeBucket( + bucket, + "blr1", + ) + if err != nil { + log.Fatal(err) + } + } - info, err := s.Client.FPutObject(ctx, bucketName, objectName, filePath, minio.PutObjectOptions{}) + // List all Spaces. + spaces, err := client.ListBuckets() if err != nil { - return err + log.Fatal(err) + } + for _, space := range spaces { + fmt.Println(space.Name) } - fmt.Println(info) + return &S3Storage{ + client: client, + bucket: bucket, + } +} + +func (a *S3Storage) Connect() error { return nil } -func (s *MinioStorage) GetSignedURL(objectName string, alias string) (string, error) { - reqParams := make(url.Values) - reqParams.Set("response-content-disposition", "attachment; filename=\""+alias+"\"") - presignedURL, err := s.Client.PresignedGetObject(context.Background(), s.BucketName, objectName, time.Minute*5, reqParams) +func (a *S3Storage) Upload(objectName string, data bytes.Buffer) error { + dataBytes := bytes.NewReader(data.Bytes()) + _, err := a.client.PutObject( + a.bucket, + objectName, + dataBytes, + dataBytes.Size(), + minio.PutObjectOptions{}, + ) + return err +} + +func (a *S3Storage) HasObject(objectName string) bool { + obj, err := a.client.StatObject(a.bucket, objectName, minio.StatObjectOptions{}) if err != nil { - return "", err + return false } + return len(obj.Key) > 0 +} - url := os.Getenv("MINIO_URL_PREFIX") + presignedURL.RequestURI() - - return url, nil +func (a *S3Storage) GetSignedURL(objectName string) (string, error) { + url, err := a.client.PresignedGetObject( + a.bucket, objectName, time.Minute*15, url.Values{}, + ) + return url.String(), err } diff --git a/storage/storage.go b/storage/storage.go index f88a847..322ed8e 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -1,7 +1,10 @@ package storage +import "bytes" + type Storage interface { Connect() error - Upload(objectName, filePath string) error - GetSignedURL(objectName, path string) error + HasObject(objectName string) bool + Upload(objectName string, data bytes.Buffer) error + GetSignedURL(objectName string) (string, error) } From 08bc4ea40d490063608426a2c914105fc18dc552 Mon Sep 17 00:00:00 2001 From: Reaper Gelera Date: Wed, 1 May 2024 16:56:30 +0530 Subject: [PATCH 3/6] ci: update versions --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 2e32932..e4659a9 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - version: [1.16, 1.17, '1.20.0'] + version: ['1.19.0','1.21.5'] steps: - uses: actions/checkout@v3 From 2f00b372289afca4abb482e93fd61161de86dffe Mon Sep 17 00:00:00 2001 From: Reaper Gelera Date: Tue, 11 Jun 2024 10:15:48 +0530 Subject: [PATCH 4/6] feat: add storage cleaning job --- .env.example | 11 +++++++---- cmd/goblin-api/main.go | 37 +++++++++++++++++++++++++++++++++++++ storage/minio.go | 24 ++++++++++++++++++++++++ storage/storage.go | 12 +++++++++++- 4 files changed, 79 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index c719501..33e1228 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,9 @@ -MINIO_ROOT_USER="" # eg: minioadmin -MINIO_ROOT_PASSWORD="" # eg: "minioadmin" -BUCKET_NAME="" # eg: "binaries" +STORAGE_CLIENT_ID= +STORAGE_CLIENT_SECRET= +STORAGE_ENDPOINT= +STORAGE_BUCKET= ORIGIN_URL="" # eg: "http://localhost" depending on your nginx config changes -MINIO_URL_PREFIX="" # eg: "http://localhost:9000" depending on your nginx config changes GITHUB_TOKEN="" # required for github version resolutions + +# uncomment the below to enable cache clearing based on the passed duration +# CLEAR_CACHE_TIME="10m" diff --git a/cmd/goblin-api/main.go b/cmd/goblin-api/main.go index 57b7d2b..c800d64 100644 --- a/cmd/goblin-api/main.go +++ b/cmd/goblin-api/main.go @@ -12,6 +12,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/barelyhuman/go/env" "github.com/barelyhuman/goblin/build" @@ -92,9 +93,45 @@ func main() { } } + clearStorageBackgroundJob() StartServer(portFlag) } +func clearStorageBackgroundJob() { + cacheHoldEnv := env.Get("CLEAR_CACHE_TIME", "") + if len(cacheHoldEnv) == 0 { + return + } + + cacheHoldDuration, _ := time.ParseDuration(cacheHoldEnv) + + cleaner := func(storageClient storage.Storage) { + log.Println("Cleaning Cached Storage Object") + objects := storageClient.ListObjects() + for _, obj := range objects { + objExpiry := obj.LastModified.Add(cacheHoldDuration) + if time.Now().Equal(objExpiry) || time.Now().After(objExpiry) { + storageClient.RemoveObject(obj.Key) + } + } + } + + ticker := time.NewTicker(cacheHoldDuration) + quit := make(chan struct{}) + + go func() { + for { + select { + case <-ticker.C: + cleaner(storageClient) + case <-quit: + ticker.Stop() + return + } + } + }() +} + func isStorageEnabled() bool { useStorageEnv := env.Get("STORAGE_ENABLED", "false") useStorage := false diff --git a/storage/minio.go b/storage/minio.go index 2869f01..4228856 100644 --- a/storage/minio.go +++ b/storage/minio.go @@ -85,3 +85,27 @@ func (a *S3Storage) GetSignedURL(objectName string) (string, error) { ) return url.String(), err } + +func (a *S3Storage) ListObjects() []MicroObject { + doneCh := make(chan struct{}) + defer close(doneCh) + recursive := true + collection := []MicroObject{} + for obj := range a.client.ListObjectsV2(a.bucket, "", recursive, doneCh) { + collection = append(collection, + MicroObject{ + LastModified: obj.LastModified, + Key: obj.Key, + }, + ) + } + return collection +} + +func (a *S3Storage) RemoveObject(objectName string) (bool, error) { + err := a.client.RemoveObject(a.bucket, objectName) + if err != nil { + return false, err + } + return true, nil +} diff --git a/storage/storage.go b/storage/storage.go index 322ed8e..46ae52a 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -1,10 +1,20 @@ package storage -import "bytes" +import ( + "bytes" + "time" +) + +type MicroObject struct { + LastModified time.Time + Key string +} type Storage interface { Connect() error HasObject(objectName string) bool Upload(objectName string, data bytes.Buffer) error GetSignedURL(objectName string) (string, error) + ListObjects() []MicroObject + RemoveObject(objectName string) (bool, error) } From b852a5fe8bd63f82210a0ed8bfba96f5059af1a8 Mon Sep 17 00:00:00 2001 From: Reaper Gelera Date: Tue, 11 Jun 2024 11:05:07 +0530 Subject: [PATCH 5/6] docs: update readme for self hosting config --- .env.example | 3 ++- readme.md | 30 +++++++++++++++++++----------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/.env.example b/.env.example index 33e1228..a36e1fe 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,4 @@ +STORAGE_ENABLED= STORAGE_CLIENT_ID= STORAGE_CLIENT_SECRET= STORAGE_ENDPOINT= @@ -6,4 +7,4 @@ ORIGIN_URL="" # eg: "http://localhost" depending on your nginx config changes GITHUB_TOKEN="" # required for github version resolutions # uncomment the below to enable cache clearing based on the passed duration -# CLEAR_CACHE_TIME="10m" +# CLEAR_CACHE_TIME="10m" diff --git a/readme.md b/readme.md index 83a08ee..878bf73 100644 --- a/readme.md +++ b/readme.md @@ -26,6 +26,7 @@ original one) - Easy to use - Users don't need go to install your CLI - Works with most common package ( Raise an [issue](/issues) if you find it not working with something) +- Persistence ( can be made into a temporary cache to minimize storage costs) - Self Hostable ## Roadmap @@ -55,9 +56,9 @@ hosted version from me at [goblin.run](https://goblin.run) you'll have to make a few tweaks to the original code to make it work in a simpler fashion** +### Existing Image -### Existing Image -The repository builds a nightly image that you can use directly if you don't wish to tweak anything. +The repository builds a nightly image that you can use directly if you don't wish to tweak anything. ```sh $ docker run -p "3000:3000" ghcr.io/barelyhuman/goblin:nightly @@ -65,15 +66,6 @@ $ docker run -p "3000:3000" ghcr.io/barelyhuman/goblin:nightly $ docker run -e "GOBLIN_ORIGIN_URL=example.com" -p "3000:3000" ghcr.io/barelyhuman/goblin:nightly ``` -### From Source -Let's start - -1. Clone the code - -```sh -git clone https://github.com/barelyhuman/goblin -``` - #### Using Docker 1. Setup docker or any other platform that would allow you to build and run @@ -144,6 +136,22 @@ VITE_GOBLIN_ORIGIN_URL= running the `build.sh` should handle building with the needed env files and restarting the server for you. +## Configuration + +The server can be configured easily using environment variables + +| KEY | default | description | options | +| --------------------- | -------------------------- | ----------------------------------------------------------------------------------- | --------------- | +| STORAGE_ENABLED | `false` | Enable persistence | `true` | +| STORAGE_CLIENT_ID | | CLIENT_ID of a S3 compatible storage | | +| STORAGE_CLIENT_SECRET | | CLIENT_SECRET of a S3 compatible storage | | +| STORAGE_ENDPOINT | | Endpoint value of an S3 compatible storage | | +| STORAGE_BUCKET | | Bucket name of the S3 compatible storage | | +| PORT | `3000` | Default port for running the application | | +| ORIGIN_URL | `http://localhost:${PORT}` | Default URL of the application | | +| GITHUB_TOKEN | | Github authenticated token for accessing github repositories and resolving versions | | +| CLEAR_CACHE_TIME | | Duration used to clear and set expiry for stored binaries | `1m`,`30s`, etc | + ## License [MIT](/LICENSE) From 467bc575d25701ef6c12bfb63ee0b1cd14716474 Mon Sep 17 00:00:00 2001 From: Reaper Gelera Date: Tue, 11 Jun 2024 11:05:58 +0530 Subject: [PATCH 6/6] docs: typo --- readme.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/readme.md b/readme.md index 878bf73..2fa4108 100644 --- a/readme.md +++ b/readme.md @@ -140,17 +140,17 @@ restarting the server for you. The server can be configured easily using environment variables -| KEY | default | description | options | -| --------------------- | -------------------------- | ----------------------------------------------------------------------------------- | --------------- | -| STORAGE_ENABLED | `false` | Enable persistence | `true` | -| STORAGE_CLIENT_ID | | CLIENT_ID of a S3 compatible storage | | -| STORAGE_CLIENT_SECRET | | CLIENT_SECRET of a S3 compatible storage | | -| STORAGE_ENDPOINT | | Endpoint value of an S3 compatible storage | | -| STORAGE_BUCKET | | Bucket name of the S3 compatible storage | | -| PORT | `3000` | Default port for running the application | | -| ORIGIN_URL | `http://localhost:${PORT}` | Default URL of the application | | -| GITHUB_TOKEN | | Github authenticated token for accessing github repositories and resolving versions | | -| CLEAR_CACHE_TIME | | Duration used to clear and set expiry for stored binaries | `1m`,`30s`, etc | +| KEY | default | description | options | +| --------------------- | -------------------------- | ------------------------------------------------------------------------------------ | --------------- | +| STORAGE_ENABLED | `false` | Enable persistence | `true` | +| STORAGE_CLIENT_ID | | CLIENT_ID of a S3 compatible storage | | +| STORAGE_CLIENT_SECRET | | CLIENT_SECRET of a S3 compatible storage | | +| STORAGE_ENDPOINT | | Endpoint value of an S3 compatible storage | | +| STORAGE_BUCKET | | Bucket name of the S3 compatible storage | | +| PORT | `3000` | Default port for running the application | | +| ORIGIN_URL | `http://localhost:${PORT}` | Default URL of the application | | +| GITHUB_TOKEN | | Github authentication token for accessing github repositories and resolving versions | | +| CLEAR_CACHE_TIME | | Duration used to clear and set expiry for stored binaries | `1m`,`30s`, etc | ## License