Add caching for fluxv2 plugin (first step toward ##3032) (#3044)

* bump fluxv2 version for local dev env

* step 2

* step 3

* step 4

* step 5

* step 6

* step 7

* step 8

* step 9

* step 10

* step 11

* step 12

* step 13

* step 14

* undo un-intended changes from a messed up merge

* step 15

* step 15

* test cleanup

* Michael's feedback

* Michael's comments #2

* step 16

* step 17

* step 18

* revert unintended change

* step 19

* Michael's feedback #3

* Michael's feedback #4

* small change to force CI test run

* small change to add a name to AvailablePackageDetail response

* Michael's feedback #5

* add a couple of debug statements to help diagnose CI failures

* Add --set redis.auth.password=password in CI

* Fix chart deps version

Co-authored-by: Antonio Gamez Diaz <agamez@vmware.com>

chart/kubeapps/Chart.lock

@@ -4,6 +4,9 @@ dependencies:
   version: 1.7.0
 - name: postgresql
   repository: https://charts.bitnami.com/bitnami
-  version: 10.5.1
-digest: sha256:7a795813de94aae4ab59043b912c3f4f2c093569061dcfe4fb4b752d07a46539
-generated: "2021-07-05T14:02:53.640429943+02:00"
+  version: 10.5.2
+- name: redis
+  repository: https://charts.bitnami.com/bitnami
+  version: 14.6.6
+digest: sha256:36eb43f0883184fd54a4e61e389c2e4c76ad6b0c3def3a94661388a04b417b3b
+generated: "2021-07-07T20:21:11.869018729+02:00"

chart/kubeapps/Chart.yaml

@@ -11,6 +11,9 @@ dependencies:
   - name: postgresql
     repository: https://charts.bitnami.com/bitnami
     version: 10.x.x
+  - name: redis
+    repository: https://charts.bitnami.com/bitnami
+    version: 14.x.x
 description: Kubeapps is a dashboard for your Kubernetes cluster that makes it easy to deploy and manage applications in your cluster using Helm
 home: https://kubeapps.com
 icon: https://raw.githubusercontent.com/kubeapps/kubeapps/master/docs/img/logo.png

chart/kubeapps/templates/NOTES.txt

@@ -1,5 +1,7 @@
 {{- $postgresqlSecretName := include "kubeapps.postgresql.secretName" . -}}
 
+{{- $redisSecretName := include "kubeapps.redis.secretName" . -}}
+
 ** Please be patient while the chart is being deployed **
 
 Tip:
@@ -65,11 +67,20 @@ To access Kubeapps from outside your K8s cluster, follow the steps below:
 ##########################################################################################################
 {{- end }}
 
+{{ if and (.Values.featureFlags.kubeappsAPIsServer) (not .Values.redis.existingSecret) (empty .Values.redis.redisPassword) -}}
+##########################################################################################################
+### WARNING: You did not provide a value for the redisPassword so one has been generated randomly      ###
+##########################################################################################################
+{{- end }}
+
 {{- $passwordValidationErrors := list -}}
 
 {{- $postgresqlPasswordValidationErrors := include "common.validations.values.postgresql.passwords" (dict "secret" $postgresqlSecretName "subchart" true "context" $) -}}
 {{- $passwordValidationErrors = append $passwordValidationErrors $postgresqlPasswordValidationErrors -}}
 
+{{- $redisPasswordValidationErrors := include "common.validations.values.redis.passwords" (dict "secret" $redisSecretName "subchart" true "context" $) -}}
+{{- $passwordValidationErrors = append $passwordValidationErrors $redisPasswordValidationErrors -}}
+
 {{- include "common.errors.upgrade.passwords.empty" (dict "validationErrors" $passwordValidationErrors "context" $) -}}
 {{- include "kubeapps.checkRollingTags" . }}
 {{- include "kubeapps.validateValues" . }}

chart/kubeapps/templates/_helpers.tpl

@@ -16,6 +16,15 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+{{/*
+Create a default fully qualified app name for Redis dependency.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "kubeapps.redis.fullname" -}}
+{{- $name := default "redis" .Values.redis.nameOverride -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
 {{/*
 Create name for the apprepository-controller based on the fullname
 */}}
@@ -166,6 +175,17 @@ Return the Postgresql secret name
   {{- end -}}
 {{- end -}}
 
+{{/*
+Return the Redis secret name
+*/}}
+{{- define "kubeapps.redis.secretName" -}}
+  {{- if .Values.redis.existingSecret }}
+      {{- printf "%s" .Values.redis.existingSecret -}}
+  {{- else -}}
+      {{- printf "%s" (include "kubeapps.redis.fullname" .) -}}
+  {{- end -}}
+{{- end -}}
+
 {{/*
 Compile all warnings into a single message, and call fail.
 */}}

chart/kubeapps/templates/kubeappsapis/deployment.yaml

@@ -85,6 +85,19 @@ spec:
           env:
             - name: PORT
               value: {{ .Values.kubeappsapis.containerPort | quote }}
+            # REDIS-* vars are required by the plugins for caching functionality
+            # TODO (gfichtenolt) this as required by the kubeapps apis service (which will 
+            # longer-term pass something to the plugins so that the plugins won't need to 
+            # know these details). Currently they're used directly by the flux plugin
+            - name: REDIS_ADDR
+              value: kubeapps-redis-master.{{ .Release.Namespace }}.svc.cluster.local:6379
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  key: redis-password
+                  name: {{ include "kubeapps.redis.secretName" . }}
+            - name: REDIS_DB
+              value: "0"
             # TODO(agamez): pass this configuration using a separated config file
             # These env vars are currently (and temporarily) required by the 'helm' plugin
             - name: POD_NAMESPACE

cmd/asset-syncer/utils.go

@@ -785,34 +785,25 @@ func (f *fileImporter) fetchAndImportIcon(c models.Chart, r *models.RepoInternal
 		return nil
 	}
 
-	req, err := http.NewRequest("GET", c.Icon, nil)
-	if err != nil {
-		return err
-	}
-	req.Header.Set("User-Agent", userAgent())
+	reqHeaders := make(map[string]string)
+	reqHeaders["User-Agent"] = userAgent()
 	if passCredentials || len(r.AuthorizationHeader) > 0 && isURLDomainEqual(c.Icon, r.URL) {
-		req.Header.Set("Authorization", r.AuthorizationHeader)
+		reqHeaders["Authorization"] = r.AuthorizationHeader
 	}
 
-	res, err := f.netClient.Do(req)
-	if res != nil {
-		defer res.Body.Close()
+	reader, contentType, err := httpclient.GetStream(c.Icon, f.netClient, reqHeaders)
+	if reader != nil {
+		defer reader.Close()
 	}
 	if err != nil {
 		return err
 	}
 
-	if res.StatusCode != http.StatusOK {
-		return fmt.Errorf("%d %s", res.StatusCode, c.Icon)
-	}
-
-	b := []byte{}
-	contentType := ""
 	var img image.Image
 	// if the icon is in any other format try to convert it to PNG
-	if strings.Contains(res.Header.Get("Content-Type"), "image/svg") {
+	if strings.Contains(contentType, "image/svg") {
 		// if the icon is an SVG, it requires special processing
-		icon, err := oksvg.ReadIconStream(res.Body)
+		icon, err := oksvg.ReadIconStream(reader)
 		if err != nil {
 			log.WithFields(log.Fields{"name": c.Name}).WithError(err).Error("failed to decode icon")
 			return err
@@ -823,7 +814,7 @@ func (f *fileImporter) fetchAndImportIcon(c models.Chart, r *models.RepoInternal
 		icon.Draw(rasterx.NewDasher(w, h, rasterx.NewScannerGV(w, h, rgba, rgba.Bounds())), 1)
 		img = rgba
 	} else {
-		img, err = imaging.Decode(res.Body)
+		img, err = imaging.Decode(reader)
 		if err != nil {
 			log.WithFields(log.Fields{"name": c.Name}).WithError(err).Error("failed to decode icon")
 			return err
@@ -838,7 +829,7 @@ func (f *fileImporter) fetchAndImportIcon(c models.Chart, r *models.RepoInternal
 		log.WithFields(log.Fields{"name": c.Name}).WithError(err).Error("failed to encode icon")
 		return err
 	}
-	b = buf.Bytes()
+	b := buf.Bytes()
 	contentType = "image/png"
 
 	return f.manager.updateIcon(models.Repo{Namespace: r.Namespace, Name: r.Name}, b, contentType, c.ID)

cmd/asset-syncer/utils_test.go

@@ -89,13 +89,12 @@ func (h *goodAuthenticatedHTTPClient) Do(req *http.Request) (*http.Response, err
 	// Ensure we're sending an Authorization header
 	if req.Header.Get("Authorization") == "" {
 		w.WriteHeader(401)
-	}
-	// Ensure we're sending the right Authorization header
-	if !strings.Contains(req.Header.Get("Authorization"), "Bearer ThisSecretAccessTokenAuthenticatesTheClient") {
+	} else if !strings.Contains(req.Header.Get("Authorization"), "Bearer ThisSecretAccessTokenAuthenticatesTheClient") {
+		// Ensure we're sending the right Authorization header
 		w.WriteHeader(403)
+	} else {
+		w.Write(iconBytes())
 	}
-
-	w.Write(iconBytes())
 	return w.Result(), nil
 }
 
@@ -408,7 +407,7 @@ func Test_fetchAndImportIcon(t *testing.T) {
 		defer cleanup()
 		netClient := &badHTTPClient{}
 		fImporter := fileImporter{pgManager, netClient}
-		assert.Err(t, fmt.Errorf("500 %s", charts[0].Icon), fImporter.fetchAndImportIcon(charts[0], repo))
+		assert.Err(t, fmt.Errorf("GET request to [%s] failed due to status [500]", charts[0].Icon), fImporter.fetchAndImportIcon(charts[0], repo))
 	})
 
 	t.Run("bad icon", func(t *testing.T) {
@@ -457,7 +456,7 @@ func Test_fetchAndImportIcon(t *testing.T) {
 		netClient := &goodAuthenticatedHTTPClient{}
 
 		fImporter := fileImporter{pgManager, netClient}
-		assert.Err(t, fmt.Errorf("401 %s", charts[0].Icon), fImporter.fetchAndImportIcon(charts[0], repo))
+		assert.Err(t, fmt.Errorf("GET request to [%s] failed due to status [401]", charts[0].Icon), fImporter.fetchAndImportIcon(charts[0], repo))
 	})
 
 	t.Run("valid icon (not passing through the auth header)", func(t *testing.T) {
@@ -467,7 +466,7 @@ func Test_fetchAndImportIcon(t *testing.T) {
 
 		fImporter := fileImporter{pgManager, netClient}
 		passCredentials = false
-		assert.Err(t, fmt.Errorf("401 %s", charts[0].Icon), fImporter.fetchAndImportIcon(charts[0], repo))
+		assert.Err(t, fmt.Errorf("GET request to [%s] failed due to status [401]", charts[0].Icon), fImporter.fetchAndImportIcon(charts[0], repo))
 	})
 
 	t.Run("valid icon (passing through the auth header if same domain)", func(t *testing.T) {
@@ -702,7 +701,7 @@ func Test_ociAPICli(t *testing.T) {
 			},
 		}
 		_, err := apiCli.TagList("apache")
-		assert.Err(t, fmt.Errorf("request failed: forbidden"), err)
+		assert.Err(t, fmt.Errorf("GET request to [http://oci-test/v2/apache/tags/list] failed due to status [500]: forbidden"), err)
 	})
 
 	t.Run("TagList - successful request", func(t *testing.T) {
@@ -727,7 +726,7 @@ func Test_ociAPICli(t *testing.T) {
 			netClient:  &authenticatedOCIAPIHTTPClient{},
 		}
 		_, err := apiCli.TagList("apache")
-		assert.Err(t, fmt.Errorf("request failed: "), err)
+		assert.Err(t, fmt.Errorf("GET request to [http://oci-test/v2/apache/tags/list] failed due to status [500]"), err)
 	})
 
 	t.Run("TagList with auth - success", func(t *testing.T) {
@@ -752,7 +751,7 @@ func Test_ociAPICli(t *testing.T) {
 			netClient: &badHTTPClient{},
 		}
 		_, err := apiCli.IsHelmChart("apache", "7.5.1")
-		assert.Err(t, fmt.Errorf("request failed: "), err)
+		assert.Err(t, fmt.Errorf("GET request to [http://oci-test/v2/apache/manifests/7.5.1] failed due to status [500]"), err)
 	})
 
 	t.Run("IsHelmChart - successful request", func(t *testing.T) {