Merge remote-tracking branch 'origin/main' into enhancement/alert-sta…

…te-manager * origin/main: (50 commits) Bugfix: Cannot delete then recreate an env (#457) Bugfix: Get releases (#454) Missing parameter (#455) Fix status overwrite (#456) Bugfix: Gitops commit state (#451) Enhancement: Hiding secrets in logs (#453) One click guide position change More caching Go build cache test Caching go modules Git config was constantly overwritten locally.. Git config was constantly overwritten locally.. Cosmetics Encryption test must run in isolation; meddler caches the encryption parameters as it is designed to run as a singleton Re-encryption Cosmetics Fixing disabled preview build Fixing disabled preview build Bugfix: Should filter deploy configs to the existing envs (#447) Enhancement: One-click-config guide (#446) ...
gimlet-io · Mar 14, 2023 · a7cfc98 · a7cfc98
2 parents 9c24b23 + 8003a3c
commit a7cfc98
Show file tree

Hide file tree

Showing 97 changed files with 1,855 additions and 35,486 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -17,17 +17,40 @@ jobs:
       uses: actions/setup-go@v2
       with:
         go-version: 1.19
+
+    - id: go-cache-paths
+      run: |
+        echo "::set-output name=go-build::$(go env GOCACHE)"
+        echo "::set-output name=go-mod::$(go env GOMODCACHE)"
+
     - name: ⬇️ Check out code into the Go module directory
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
       with:
         fetch-depth: 5
     - name: 🤞 Test Dashboard Frontend
       run: make test-dashboard-frontend
+
+    - name: Go Build Cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.go-cache-paths.outputs.go-build }}
+        key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
+
+    - name: Go Mod Cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.go-cache-paths.outputs.go-mod }}
+        key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
+
     - name: 🤞 Test
       run: |
+        git config --global user.email "git@gimlet.io"
+        git config --global user.name "Github Actions"
         make test
     - name: 🐘 Test with Postgres store
       run: |
+        git config --global user.email "git@gimlet.io"
+        git config --global user.name "Github Actions"
         make test-with-postgres
 
   cli:
@@ -43,11 +66,24 @@ jobs:
       uses: actions/setup-node@v3
       with:
         node-version: 16.1.0
+    - id: go-cache-paths
+      run: |
+        echo "::set-output name=go-build::$(go env GOCACHE)"
+        echo "::set-output name=go-mod::$(go env GOMODCACHE)"
     - name: ⬇️ Check out code into the Go module directory
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
       with:
         fetch-depth: 5
-
+    - name: Go Build Cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.go-cache-paths.outputs.go-build }}
+        key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
+    - name: Go Mod Cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.go-cache-paths.outputs.go-mod }}
+        key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
     - name: 🏗️ Build chart configure frontend
       run: make build-cli-frontend
     - name: 🏗️ Build stack frontend
@@ -66,10 +102,24 @@ jobs:
       uses: actions/setup-go@v2
       with:
         go-version: 1.19
+    - id: go-cache-paths
+      run: |
+        echo "::set-output name=go-build::$(go env GOCACHE)"
+        echo "::set-output name=go-mod::$(go env GOMODCACHE)"
     - name: ⬇️ Check out code into the Go module directory
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
       with:
         fetch-depth: 5
+    - name: Go Build Cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.go-cache-paths.outputs.go-build }}
+        key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
+    - name: Go Mod Cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.go-cache-paths.outputs.go-mod }}
+        key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
     - name: 🏗️ Build Gimlet Agent
       run: make build-agent
       env:
@@ -84,10 +134,24 @@ jobs:
       uses: actions/setup-go@v2
       with:
         go-version: 1.19
+    - id: go-cache-paths
+      run: |
+        echo "::set-output name=go-build::$(go env GOCACHE)"
+        echo "::set-output name=go-mod::$(go env GOMODCACHE)"
     - name: ⬇️ Check out code into the Go module directory
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
       with:
         fetch-depth: 5
+    - name: Go Build Cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.go-cache-paths.outputs.go-build }}
+        key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
+    - name: Go Mod Cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.go-cache-paths.outputs.go-mod }}
+        key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
     - name: 🏗️ Build Dashboard
       run: make build-dashboard
       env:

diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml
diff --git a/Makefile b/Makefile
@@ -20,18 +20,27 @@ test-prep:
 	touch cmd/installer/web/main.css
 	touch cmd/installer/web/index.html
 	touch cmd/installer/web/favicon.ico
-	git config --global user.email "git@gimlet.io"
-	git config --global user.name "Github Actions"
 
 test: test-prep
-	go test -timeout 60s $(shell go list ./... )
+	go test -timeout 60s $(shell go list ./...)
+	go test -timeout 60s -tags=encryption github.com/gimlet-io/gimlet-cli/pkg/dashboard/store -run TestEncryption
+	go test -timeout 60s -tags=encryption github.com/gimlet-io/gimlet-cli/pkg/dashboard/store -run TestReEncryption
+	go test -timeout 60s -tags=encryption github.com/gimlet-io/gimlet-cli/pkg/dashboard/store -run TestInitEncryption
+	go test -timeout 60s -tags=encryption github.com/gimlet-io/gimlet-cli/pkg/dashboard/store -run TestUnquote
+
 test-dashboard-frontend:
 	(cd web/dashboard; npm install; npm run test)
 test-with-postgres:
 	docker stop testpostgres || true
 	docker run --rm -e POSTGRES_PASSWORD=mysecretpassword -p 5432:5432 --name testpostgres -d postgres
 
-	DATABASE_DRIVER=postgres DATABASE_CONFIG=postgres://postgres:mysecretpassword@127.0.0.1:5432/postgres?sslmode=disable go test -count=1 -timeout 60s github.com/gimlet-io/gimlet-cli/pkg/dashboard/store/...
+	export DATABASE_DRIVER=postgres
+	export DATABASE_CONFIG=postgres://postgres:mysecretpassword@127.0.0.1:5432/postgres?sslmode=disable
+
+	go test -timeout 60s -tags=encryption github.com/gimlet-io/gimlet-cli/pkg/dashboard/store -run TestEncryption
+	go test -timeout 60s -tags=encryption github.com/gimlet-io/gimlet-cli/pkg/dashboard/store -run TestReEncryption
+	go test -timeout 60s -tags=encryption github.com/gimlet-io/gimlet-cli/pkg/dashboard/store -run TestInitEncryption
+	go test -timeout 60s -tags=encryption github.com/gimlet-io/gimlet-cli/pkg/dashboard/store -run TestUnquote
 
 	docker stop testpostgres || true
 

diff --git a/README.md b/README.md
@@ -11,19 +11,6 @@
   <img alt="Gimlet" src="https://github.com/gimlet-io/gimlet-documentation/blob/main/public/logo.svg" width="200">
 </picture>
 
-
-##
-
-One off SQLs:
-
-```
-update users set admin=true where login='laszlocph';
-```
-
-- gitopsRepos and gitopsRepo env vars support needs to be done: it should bootstrap an env entry, or we deprecate it
-
-##
-
 ## Overview
 
 With Gimlet, you can build and run your developer platform on Kubernetes.
@@ -109,4 +96,3 @@ gimlet manifest template -f staging.yaml -o manifests.yaml
 ## Youtube playlist on building a full developer platform
 
 [Building a developer platform on CIVO Cloud with Gimlet and gitops](https://youtube.com/playlist?list=PLjJkiSWPwuPJeIEOn5BWMFdxSSpiQPQ4P)
-
diff --git a/cmd/dashboard/config/config.go b/cmd/dashboard/config/config.go
@@ -114,8 +114,10 @@ type Chart struct {
 }
 
 type Database struct {
-	Driver string `envconfig:"DATABASE_DRIVER"`
-	Config string `envconfig:"DATABASE_CONFIG"`
+	Driver           string `envconfig:"DATABASE_DRIVER"`
+	Config           string `envconfig:"DATABASE_CONFIG"`
+	EncryptionKey    string `envconfig:"DATABASE_ENCRYPTION_KEY"`
+	EncryptionKeyNew string `envconfig:"DATABASE_ENCRYPTION_KEY_NEW"`
 }
 
 type Notifications struct {

diff --git a/cmd/dashboard/dashboard.go b/cmd/dashboard/dashboard.go
@@ -58,7 +58,17 @@ func main() {
 	agentWSHub := streaming.NewAgentWSHub(*clientHub)
 	go agentWSHub.Run()
 
-	store := store.New(config.Database.Driver, config.Database.Config)
+	store := store.New(
+		config.Database.Driver,
+		config.Database.Config,
+		config.Database.EncryptionKey,
+		config.Database.EncryptionKeyNew,
+	)
+
+	err = reencrypt(store, config.Database.EncryptionKeyNew)
+	if err != nil {
+		panic(err)
+	}
 
 	err = setupAdminUser(config, store)
 	if err != nil {
@@ -138,6 +148,7 @@ func main() {
 			Releases:  releases,
 			Perf:      perf,
 			Store:     store,
+			Config:    config,
 		}
 		go releaseStateWorker.Run()
 	}
@@ -153,6 +164,9 @@ func main() {
 	metricsRouter.Get("/metrics", promhttp.Handler().ServeHTTP)
 	go http.ListenAndServe(":9001", metricsRouter)
 
+	logger := log.New()
+	logger.Formatter = &customFormatter{}
+
 	r := server.SetupRouter(
 		config,
 		agentHub,
@@ -165,6 +179,7 @@ func main() {
 		alertStateManager,
 		notificationsManager,
 		perf,
+		logger,
 	)
 
 	go func() {

diff --git a/cmd/dashboard/dashboard_test.go b/cmd/dashboard/dashboard_test.go
@@ -33,7 +33,9 @@ func TestParseEnvs_Empty(t *testing.T) {
 }
 
 func TestBootstrapEnvs(t *testing.T) {
-	s := store.NewTest()
+	encryptionKey := "the-key-has-to-be-32-bytes-long!"
+	encryptionKeyNew := ""
+	s := store.NewTest(encryptionKey, encryptionKeyNew)
 	defer func() {
 		s.Close()
 	}()

diff --git a/cmd/dashboard/init.go b/cmd/dashboard/init.go
@@ -4,8 +4,11 @@ import (
 	"database/sql"
 	"encoding/base32"
 	"fmt"
+	"os"
 	"path"
+	"regexp"
 	"runtime"
+	"time"
 
 	"github.com/gimlet-io/gimlet-cli/cmd/dashboard/config"
 	"github.com/gimlet-io/gimlet-cli/pkg/dashboard/model"
@@ -138,3 +141,48 @@ func initLogger(c *config.Config) {
 		logrus.SetLevel(logrus.TraceLevel)
 	}
 }
+
+func reencrypt(store *store.Store, encryptionKeyNew string) error {
+	if encryptionKeyNew == "" {
+		return nil
+	}
+
+	users, err := store.EverytingFromUsers()
+	if err != nil {
+		return err
+	}
+	for _, u := range users {
+		store.UpdateUser(u)
+		if err != nil {
+			return err
+		}
+	}
+
+	fmt.Println("db field re-encryption is done, please replace the value of DATABASE_ENCRYPTION_KEY with the value of DATABASE_ENCRYPTION_KEY_NEW, and delete DATABASE_ENCRYPTION_KEY_NEW environment variable")
+	os.Exit(0)
+	return nil
+}
+
+type customFormatter struct{}
+
+func (f *customFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+	message := entry.Message
+	entry.Message = hideAccessToken(message)
+
+	formatter := &logrus.TextFormatter{
+		FullTimestamp:   true,
+		TimestampFormat: time.RFC3339,
+	}
+	formatted, err := formatter.Format(entry)
+
+	return []byte(formatted), err
+}
+
+func hideAccessToken(message string) string {
+	pattern := `access_token=([^& ]+)`
+	r, err := regexp.Compile(pattern)
+	if err != nil {
+		return message
+	}
+	return r.ReplaceAllString(message, "access_token=***")
+}
diff --git a/cmd/installer/main.go b/cmd/installer/main.go
@@ -593,6 +593,7 @@ func bootstrap(w http.ResponseWriter, r *http.Request) {
 		repoPerEnv,
 		installationToken,
 		true,
+		true,
 		data.scmURL,
 	)
 	if err != nil {
@@ -605,6 +606,7 @@ func bootstrap(w http.ResponseWriter, r *http.Request) {
 		repoPerEnv,
 		installationToken,
 		false,
+		false,
 		data.scmURL,
 	)
 	if err != nil {