[Backport release-1.7] Feat: The vela-apiserver supports displaying c…

…hart values stored in the OCI registry (#5509) * support helm chart values Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com> rebase Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com> no lint Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com> fix lint error Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com> add test and deprecated API Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com> fix url bug Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com> fix tests panic Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com> fix tests Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com> (cherry picked from commit fc1d8c2) * fix golint Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com> (cherry picked from commit 6c462b7) * return values.yaml Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com> (cherry picked from commit 70d8cc5) * fix test Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com> (cherry picked from commit ef4574a) * fix return values Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com> (cherry picked from commit c1488ee) * add multiple valeus yaml in Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com> (cherry picked from commit 0443618) * add old interface back Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com> (cherry picked from commit ae0ac9f) * fix golint Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com> fix test Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com> (cherry picked from commit 0095774) --------- Co-authored-by: 楚岳 <wangyike.wyk@alibaba-inc.com>
kubevela · Feb 15, 2023 · f3cdbcf · f3cdbcf
1 parent 7bd2cf4
commit f3cdbcf
Show file tree

Hide file tree

Showing 7 changed files with 244 additions and 21 deletions.
diff --git a/pkg/apiserver/domain/service/helm.go b/pkg/apiserver/domain/service/helm.go
@@ -45,8 +45,9 @@ func NewHelmService() HelmService {
 type HelmService interface {
 	ListChartNames(ctx context.Context, url string, secretName string, skipCache bool) ([]string, error)
 	ListChartVersions(ctx context.Context, url string, chartName string, secretName string, skipCache bool) (repo.ChartVersions, error)
-	GetChartValues(ctx context.Context, url string, chartName string, version string, secretName string, skipCache bool) (map[string]interface{}, error)
+	ListChartValuesFiles(ctx context.Context, url string, chartName string, version string, secretName string, repoType string, skipCache bool) (map[string]string, error)
 	ListChartRepo(ctx context.Context, projectName string) (*v1.ChartRepoResponseList, error)
+	GetChartValues(ctx context.Context, repoURL string, chartName string, version string, secretName string, repoType string, skipCache bool) (map[string]interface{}, error)
 }
 
 type defaultHelmImpl struct {
@@ -99,7 +100,7 @@ func (d defaultHelmImpl) ListChartVersions(ctx context.Context, repoURL string,
 	return chartVersions, nil
 }
 
-func (d defaultHelmImpl) GetChartValues(ctx context.Context, repoURL string, chartName string, version string, secretName string, skipCache bool) (map[string]interface{}, error) {
+func (d defaultHelmImpl) ListChartValuesFiles(ctx context.Context, repoURL string, chartName string, version string, secretName string, repoType string, skipCache bool) (map[string]string, error) {
 	if !utils.IsValidURL(repoURL) {
 		return nil, bcode.ErrRepoInvalidURL
 	}
@@ -111,13 +112,33 @@ func (d defaultHelmImpl) GetChartValues(ctx context.Context, repoURL string, cha
 			return nil, bcode.ErrRepoBasicAuth
 		}
 	}
-	v, err := d.helper.GetValuesFromChart(repoURL, chartName, version, skipCache, opts)
+	v, err := d.helper.GetValuesFromChart(repoURL, chartName, version, skipCache, repoType, opts)
 	if err != nil {
 		klog.Errorf("cannot fetch chart values repo: %s, chart: %s, version: %s, error: %s", utils.Sanitize(repoURL), utils.Sanitize(chartName), utils.Sanitize(version), err.Error())
 		return nil, bcode.ErrGetChartValues
 	}
-	res := make(map[string]interface{}, len(v))
-	flattenKey("", v, res)
+	return v.Data, nil
+}
+
+func (d defaultHelmImpl) GetChartValues(ctx context.Context, repoURL string, chartName string, version string, secretName string, repoType string, skipCache bool) (map[string]interface{}, error) {
+	if !utils.IsValidURL(repoURL) {
+		return nil, bcode.ErrRepoInvalidURL
+	}
+	var opts *common.HTTPOption
+	var err error
+	if len(secretName) != 0 {
+		opts, err = helm.SetHTTPOption(ctx, d.K8sClient, types2.NamespacedName{Namespace: types.DefaultKubeVelaNS, Name: secretName})
+		if err != nil {
+			return nil, bcode.ErrRepoBasicAuth
+		}
+	}
+	v, err := d.helper.GetValuesFromChart(repoURL, chartName, version, skipCache, repoType, opts)
+	if err != nil {
+		klog.Errorf("cannot fetch chart values repo: %s, chart: %s, version: %s, error: %s", utils.Sanitize(repoURL), utils.Sanitize(chartName), utils.Sanitize(version), err.Error())
+		return nil, bcode.ErrGetChartValues
+	}
+	res := make(map[string]interface{}, len(v.Values))
+	flattenKey("", v.Values, res)
 	return res, nil
 }
 

diff --git a/pkg/apiserver/domain/service/helm_test.go b/pkg/apiserver/domain/service/helm_test.go
@@ -213,7 +213,7 @@ var _ = Describe("test helm usecasae", func() {
 		Expect(len(versions)).Should(BeEquivalentTo(1))
 		Expect(versions[0].Version).Should(BeEquivalentTo("8.8.23"))
 
-		values, err := u.GetChartValues(ctx, mockServer.URL, "mysql", "8.8.23", "repo-secret", false)
+		values, err := u.ListChartValuesFiles(ctx, mockServer.URL, "mysql", "8.8.23", "repo-secret", "helm", false)
 		Expect(err).Should(BeNil())
 		Expect(values).ShouldNot(BeNil())
 		Expect(len(values)).ShouldNot(BeEquivalentTo(0))
@@ -228,7 +228,7 @@ var _ = Describe("test helm usecasae", func() {
 		_, err = u.ListChartVersions(ctx, "http://127.0.0.1:8080", "mysql", "repo-secret-notExist", false)
 		Expect(err).ShouldNot(BeNil())
 
-		_, err = u.GetChartValues(ctx, "http://127.0.0.1:8080", "mysql", "8.8.23", "repo-secret-notExist", false)
+		_, err = u.ListChartValuesFiles(ctx, "http://127.0.0.1:8080", "mysql", "8.8.23", "repo-secret-notExist", "helm", false)
 		Expect(err).ShouldNot(BeNil())
 	})
 })

diff --git a/pkg/apiserver/interfaces/api/repository.go b/pkg/apiserver/interfaces/api/repository.go
@@ -69,24 +69,46 @@ func (h repository) GetWebServiceRoute() *restful.WebService {
 		Writes([]string{}))
 
 	// List available chart versions
-	ws.Route(ws.GET("/charts/{chart}/versions").To(h.listVersions).
+	ws.Route(ws.GET("/chart/versions").To(h.listVersionsFromQuery).
 		Doc("list versions").
 		Metadata(restfulspec.KeyOpenAPITags, tags).
+		Param(ws.QueryParameter("chart", "helm chart").DataType("string").Required(true)).
+		Param(ws.QueryParameter("repoUrl", "helm repository url").DataType("string").Required(true)).
+		Param(ws.QueryParameter("secretName", "secret of the repo").DataType("string")).
+		Returns(200, "OK", v1.ChartVersionListResponse{}).
+		Returns(400, "Bad Request", bcode.Bcode{}).
+		Writes([]string{}))
+
+	ws.Route(ws.GET("/charts/{chart}/versions").To(h.listChartVersions).
+		Doc("list versions").Deprecate().
+		Metadata(restfulspec.KeyOpenAPITags, tags).
 		Param(ws.QueryParameter("repoUrl", "helm repository url").DataType("string")).
 		Param(ws.QueryParameter("secretName", "secret of the repo").DataType("string")).
 		Returns(200, "OK", v1.ChartVersionListResponse{}).
 		Returns(400, "Bad Request", bcode.Bcode{}).
 		Writes([]string{}))
 
 	// List available chart versions
-	ws.Route(ws.GET("/charts/{chart}/versions/{version}/values").To(h.chartValues).
+	ws.Route(ws.GET("/chart/values").To(h.chartValues).
 		Doc("get chart value").
 		Metadata(restfulspec.KeyOpenAPITags, tags).
+		Param(ws.QueryParameter("chart", "helm chart").DataType("string").Required(true)).
+		Param(ws.QueryParameter("version", "helm chart version").DataType("string").Required(true)).
+		Param(ws.QueryParameter("repoUrl", "helm repository url").DataType("string").Required(true)).
+		Param(ws.QueryParameter("repoType", "helm repository type").DataType("string").Required(true)).
+		Param(ws.QueryParameter("secretName", "secret of the repo").DataType("string")).
+		Returns(200, "OK", "").
+		Returns(400, "Bad Request", bcode.Bcode{}).
+		Writes(map[string]string{}))
+
+	ws.Route(ws.GET("/charts/{chart}/versions/{version}/values").To(h.getChartValues).
+		Doc("get chart value").Deprecate().
+		Metadata(restfulspec.KeyOpenAPITags, tags).
 		Param(ws.QueryParameter("repoUrl", "helm repository url").DataType("string")).
 		Param(ws.QueryParameter("secretName", "secret of the repo").DataType("string")).
 		Returns(200, "OK", map[string]interface{}{}).
 		Returns(400, "Bad Request", bcode.Bcode{}).
-		Writes([]string{}))
+		Writes(map[string]interface{}{}))
 
 	ws.Route(ws.GET("/image/repos").To(h.getImageRepos).
 		Doc("get the oci repos").
@@ -132,9 +154,9 @@ func (h repository) listCharts(req *restful.Request, res *restful.Response) {
 	}
 }
 
-func (h repository) listVersions(req *restful.Request, res *restful.Response) {
+func (h repository) listVersionsFromQuery(req *restful.Request, res *restful.Response) {
 	url := req.QueryParameter("repoUrl")
-	chartName := req.PathParameter("chart")
+	chartName := req.QueryParameter("chart")
 	secName := req.QueryParameter("secretName")
 	skipCache, err := isSkipCache(req)
 	if err != nil {
@@ -154,7 +176,7 @@ func (h repository) listVersions(req *restful.Request, res *restful.Response) {
 	}
 }
 
-func (h repository) chartValues(req *restful.Request, res *restful.Response) {
+func (h repository) getChartValues(req *restful.Request, res *restful.Response) {
 	url := req.QueryParameter("repoUrl")
 	secName := req.QueryParameter("secretName")
 	chartName := req.PathParameter("chart")
@@ -165,12 +187,57 @@ func (h repository) chartValues(req *restful.Request, res *restful.Response) {
 		return
 	}
 
-	versions, err := h.HelmService.GetChartValues(req.Request.Context(), url, chartName, version, secName, skipCache)
+	values, err := h.HelmService.GetChartValues(req.Request.Context(), url, chartName, version, secName, "helm", skipCache)
+	if err != nil {
+		bcode.ReturnError(req, res, err)
+		return
+	}
+	err = res.WriteEntity(values)
+	if err != nil {
+		bcode.ReturnError(req, res, err)
+		return
+	}
+}
+
+func (h repository) listChartVersions(req *restful.Request, res *restful.Response) {
+	url := req.QueryParameter("repoUrl")
+	chartName := req.PathParameter("chart")
+	secName := req.QueryParameter("secretName")
+	skipCache, err := isSkipCache(req)
+	if err != nil {
+		bcode.ReturnError(req, res, bcode.ErrSkipCacheParameter)
+		return
+	}
+	versions, err := h.HelmService.ListChartVersions(req.Request.Context(), url, chartName, secName, skipCache)
+	if err != nil {
+		bcode.ReturnError(req, res, err)
+		return
+	}
+	err = res.WriteEntity(v1.ChartVersionListResponse{Versions: versions})
+	if err != nil {
+		bcode.ReturnError(req, res, err)
+		return
+	}
+}
+
+func (h repository) chartValues(req *restful.Request, res *restful.Response) {
+	url := req.QueryParameter("repoUrl")
+	secName := req.QueryParameter("secretName")
+	chartName := req.QueryParameter("chart")
+	version := req.QueryParameter("version")
+	repoType := req.QueryParameter("repoType")
+	skipCache, err := isSkipCache(req)
+	if err != nil {
+		bcode.ReturnError(req, res, bcode.ErrSkipCacheParameter)
+		return
+	}
+
+	values, err := h.HelmService.ListChartValuesFiles(req.Request.Context(), url, chartName, version, secName, repoType, skipCache)
 	if err != nil {
 		bcode.ReturnError(req, res, err)
 		return
 	}
-	err = res.WriteEntity(versions)
+	err = res.WriteEntity(values)
 	if err != nil {
 		bcode.ReturnError(req, res, err)
 		return

diff --git a/pkg/utils/helm/helm_helper.go b/pkg/utils/helm/helm_helper.go
@@ -24,6 +24,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
@@ -32,6 +33,9 @@ import (
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/chart/loader"
 	"helm.sh/helm/v3/pkg/chartutil"
+	"helm.sh/helm/v3/pkg/cli"
+	"helm.sh/helm/v3/pkg/downloader"
+	"helm.sh/helm/v3/pkg/getter"
 	"helm.sh/helm/v3/pkg/kube"
 	"helm.sh/helm/v3/pkg/release"
 	relutil "helm.sh/helm/v3/pkg/releaseutil"
@@ -56,6 +60,12 @@ const (
 	valuesPatten = "repoUrl: %s, chart: %s, version: %s"
 )
 
+// ChartValues contain all values files in chart and default chart values
+type ChartValues struct {
+	Data   map[string]string
+	Values map[string]interface{}
+}
+
 // Helper provides helper functions for common Helm operations
 type Helper struct {
 	cache *utils2.MemoryCacheStore
@@ -309,11 +319,21 @@ func (h *Helper) ListChartsFromRepo(repoURL string, skipCache bool, opts *common
 }
 
 // GetValuesFromChart will extract the parameter from a helm chart
-func (h *Helper) GetValuesFromChart(repoURL string, chartName string, version string, skipCache bool, opts *common.HTTPOption) (map[string]interface{}, error) {
+func (h *Helper) GetValuesFromChart(repoURL string, chartName string, version string, skipCache bool, repoType string, opts *common.HTTPOption) (*ChartValues, error) {
 	if h.cache != nil && !skipCache {
 		if v := h.cache.Get(fmt.Sprintf(valuesPatten, repoURL, chartName, version)); v != nil {
-			return v.(map[string]interface{}), nil
+			return v.(*ChartValues), nil
+		}
+	}
+	if repoType == "oci" {
+		v, err := fetchChartValuesFromOciRepo(repoURL, chartName, version, opts)
+		if err != nil {
+			return nil, err
+		}
+		if h.cache != nil {
+			h.cache.Put(fmt.Sprintf(valuesPatten, repoURL, chartName, version), v, 20*time.Minute)
 		}
+		return v, nil
 	}
 	i, err := h.GetIndexInfo(repoURL, skipCache, opts)
 	if err != nil {
@@ -334,10 +354,17 @@ func (h *Helper) GetValuesFromChart(repoURL string, chartName string, version st
 		if err != nil {
 			continue
 		}
+		v := &ChartValues{
+			Data:   loadValuesYamlFile(c),
+			Values: c.Values,
+		}
+		if err != nil {
+			return nil, err
+		}
 		if h.cache != nil {
-			h.cache.Put(fmt.Sprintf(valuesPatten, repoURL, chartName, version), c.Values, calculateCacheTimeFromIndex(len(i.Entries)))
+			h.cache.Put(fmt.Sprintf(valuesPatten, repoURL, chartName, version), v, calculateCacheTimeFromIndex(len(i.Entries)))
 		}
-		return c.Values, nil
+		return v, nil
 	}
 	return nil, fmt.Errorf("cannot load chart from chart repo")
 }
@@ -351,3 +378,49 @@ func calculateCacheTimeFromIndex(length int) time.Duration {
 	}
 	return cacheTime
 }
+
+// nolint
+func fetchChartValuesFromOciRepo(repoURL string, chartName string, version string, opts *common.HTTPOption) (*ChartValues, error) {
+	d := downloader.ChartDownloader{
+		Verify:  downloader.VerifyNever,
+		Getters: getter.All(cli.New()),
+	}
+
+	if opts != nil {
+		d.Options = append(d.Options, getter.WithInsecureSkipVerifyTLS(opts.InsecureSkipTLS),
+			getter.WithTLSClientConfig(opts.CertFile, opts.KeyFile, opts.CaFile),
+			getter.WithBasicAuth(opts.Username, opts.Password))
+	}
+
+	var err error
+	dest, err := os.MkdirTemp("", "helm-")
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to fetch values file")
+	}
+	defer os.RemoveAll(dest)
+
+	chartRef := fmt.Sprintf("%s/%s", repoURL, chartName)
+	saved, _, err := d.DownloadTo(chartRef, version, dest)
+	if err != nil {
+		return nil, err
+	}
+	c, err := loader.Load(saved)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to fetch values file")
+	}
+	return &ChartValues{
+		Data:   loadValuesYamlFile(c),
+		Values: c.Values,
+	}, nil
+}
+
+func loadValuesYamlFile(chart *chart.Chart) map[string]string {
+	result := map[string]string{}
+	re := regexp.MustCompile(`.*yaml$`)
+	for _, f := range chart.Raw {
+		if re.MatchString(f.Name) && !strings.Contains(f.Name, "/") && f.Name != "Chart.yaml" {
+			result[f.Name] = string(f.Data)
+		}
+	}
+	return result
+}
diff --git a/pkg/utils/helm/helm_helper_test.go b/pkg/utils/helm/helm_helper_test.go
@@ -114,9 +114,9 @@ var _ = Describe("Test helm helper", func() {
 
 	It("Test getValues from chart", func() {
 		helper := NewHelper()
-		values, err := helper.GetValuesFromChart("./testdata", "autoscalertrait", "0.2.0", true, nil)
+		values, err := helper.GetValuesFromChart("./testdata", "autoscalertrait", "0.2.0", true, "helm", nil)
 		Expect(err).Should(BeNil())
-		Expect(values).ShouldNot(BeEmpty())
+		Expect(values).ShouldNot(BeNil())
 	})
 })
 

diff --git a/test/e2e-apiserver-test/repository_test.go b/test/e2e-apiserver-test/repository_test.go
@@ -0,0 +1,42 @@
+/*
+Copyright 2022 The KubeVela Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_apiserver_test
+
+import (
+	"io"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Helm rest api test", func() {
+
+	Describe("helm repo api test", func() {
+		It("test fetching chart values in OCI registry", func() {
+			resp := getWithQuery("/repository/chart/values", map[string]string{
+				"repoUrl":  "oci://ghcr.io",
+				"chart":    "stefanprodan/charts/podinfo",
+				"repoType": "oci",
+				"version":  "6.1.0",
+			})
+			defer resp.Body.Close()
+			values, err := io.ReadAll(resp.Body)
+			Expect(err).Should(BeNil())
+			Expect(len(values)).ShouldNot(BeEquivalentTo(0))
+		})
+	})
+})