From fe57de64115c6e9d9e66bbf0f6c1aed4c8082cb2 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Sat, 15 Nov 2025 21:46:32 +0100 Subject: [PATCH 1/3] [Feature] [Platform] Install multi type support --- .gitignore | 4 +- CHANGELOG.md | 1 + docs/platform.install.md | 87 +++++++++++++++++++++++ go.mod | 12 ++-- go.sum | 13 ++++ internal/docs_test.go | 109 ++++++++++++++++++++--------- pkg/license_manager/client.go | 9 +++ pkg/platform/pack/cache.go | 2 +- pkg/platform/pack/export.go | 21 ++---- pkg/platform/pack/import.go | 10 ++- pkg/platform/pack/utils.go | 91 ++++++++++++++++++++++-- pkg/platform/package_install.go | 7 +- pkg/platform/package_test.go | 9 +-- pkg/util/cli/flag.go | 23 ++++-- pkg/util/cli/lm.go | 15 +++- pkg/util/cli/registry.go | 5 +- pkg/util/k8sutil/helm/package.go | 116 +++++++++++++++++++++++++++++-- 17 files changed, 442 insertions(+), 92 deletions(-) create mode 100644 docs/platform.install.md diff --git a/.gitignore b/.gitignore index 8082bf663..a4af0fa09 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,6 @@ kustomize_test/ tools/codegen/boilerplate.go.txt .checksum.code -.checksum.mod \ No newline at end of file +.checksum.mod + +cache/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 8adc7023f..877cfbf9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - (Feature) (Platform) Do not require LM during install commands - (Feature) (Platform) ArangoRoute Redirect - (Feature) (Platform) Request ID & Header Standardization +- (Feature) (Platform) Install multi type support ## [1.3.1](https://github.com/arangodb/kube-arangodb/tree/1.3.1) (2025-10-07) - (Documentation) Add ArangoPlatformStorage Docs & Examples diff --git a/docs/platform.install.md b/docs/platform.install.md new file mode 100644 index 000000000..61c5dfeec --- /dev/null +++ b/docs/platform.install.md @@ -0,0 +1,87 @@ +--- +layout: page +parent: ArangoDBPlatform +title: Platform Installation File Schema +--- + +# Installation Definition + +## Example + +```yaml +packages: + nginx: # OCI + chart: "oci://ghcr.io/nginx/charts/nginx-ingress:2.3.1" + version: 2.3.1 + prometheus: # Helm Index + chart: "index://prometheus-community.github.io/helm-charts" + version: 1.3.1 + alertmanager: # Remote Chart + chart: "https://github.com/prometheus-community/helm-charts/releases/download/alertmanager-0.1.0/alertmanager-0.1.0.tgz" + version: "0.1.0" + local: # Local File + chart: "file:///tmp/local-0.1.0.tgz" + version: "0.1.0" + inline: # Inline + chart: "" + version: "0.2.5" + platform: # Platform LicenseManager + version: v3.0.11 +``` + +## Package + +### .package.packages.\.chart + +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.3.1/pkg/util/k8sutil/helm/package.go#L87) + +Chart defines override of the PackageSpec +It supports multiple modes: +- If undefined, LicenseManager OCI Repository is used +- If starts with `file://` chart is fetched from local FileSystem +- If starts with `http://` or `https://` chart is fetched from the remote URL +- If starts with `index://` chart is fetched using Helm YAML Index File stricture (using version and name) +- If Starts with `oci://` chart is fetched from Registry Compatible OCI Repository +- If none above match, chart is decoded using Base64 encoding + +*** + +### .package.packages.\.overrides + +Type: `array` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.3.1/pkg/util/k8sutil/helm/package.go#L90) + +Overrides defines Values to override the Helm Chart Defaults (merged with Service Overrides) + +*** + +### .package.packages.\.stage + +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.3.1/pkg/util/k8sutil/helm/package.go#L74) + +Stage defines stage used in the fetch from LicenseManager + +*** + +### .package.packages.\.version + +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.3.1/pkg/util/k8sutil/helm/package.go#L77) + +Version keeps the version of the PackageSpec + +*** + +### .package.releases.\.overrides + +Type: `array` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.3.1/pkg/util/k8sutil/helm/package.go#L169) + +Overrides defines Values to override the Helm Chart Defaults during installation + +*** + +### .package.releases.\.package + +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.3.1/pkg/util/k8sutil/helm/package.go#L166) + +Package keeps the name of the Chart used from the installation script. +References to value provided in Packages + diff --git a/go.mod b/go.mod index 9956d78af..d92617ce7 100644 --- a/go.mod +++ b/go.mod @@ -53,11 +53,11 @@ require ( github.com/prometheus/prom2json v1.3.3 github.com/robfig/cron v1.2.0 github.com/rs/zerolog v1.33.0 - github.com/spf13/cobra v1.9.1 - github.com/spf13/pflag v1.0.6 + github.com/spf13/cobra v1.10.1 + github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.10.0 golang.org/x/sync v0.17.0 - golang.org/x/sys v0.37.0 + golang.org/x/sys v0.38.0 golang.org/x/text v0.30.0 golang.org/x/time v0.11.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 @@ -84,7 +84,7 @@ require ( github.com/golang-jwt/jwt/v5 v5.2.2 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 github.com/jedib0t/go-pretty/v6 v6.6.5 - github.com/regclient/regclient v0.8.3 + github.com/regclient/regclient v0.10.0 golang.org/x/oauth2 v0.30.0 google.golang.org/api v0.235.0 google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b @@ -181,7 +181,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kkdai/maglev v0.2.0 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.1 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect @@ -223,7 +223,7 @@ require ( github.com/stretchr/objx v0.5.2 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect - github.com/ulikunitz/xz v0.5.12 // indirect + github.com/ulikunitz/xz v0.5.15 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect diff --git a/go.sum b/go.sum index 057212663..97cefcf26 100644 --- a/go.sum +++ b/go.sum @@ -328,6 +328,8 @@ github.com/kkdai/maglev v0.2.0 h1:w6DCW0kAA6fstZqXkrBrlgIC3jeIRXkjOYea/m6EK/Y= github.com/kkdai/maglev v0.2.0/go.mod h1:d+mt8Lmt3uqi9aRb/BnPjzD0fy+ETs1vVXiGRnqHVZ4= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= +github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= @@ -456,6 +458,8 @@ github.com/prometheus/prom2json v1.3.3 h1:IYfSMiZ7sSOfliBoo89PcufjWO4eAR0gznGcET github.com/prometheus/prom2json v1.3.3/go.mod h1:Pv4yIPktEkK7btWsrUTWDDDrnpUrAELaOCj+oFwlgmc= github.com/regclient/regclient v0.8.3 h1:AFAPu/vmOYGyY22AIgzdBUKbzH+83lEpRioRYJ/reCs= github.com/regclient/regclient v0.8.3/go.mod h1:gjQh5uBVZoo/CngchghtQh9Hx81HOMKRRDd5WPcPkbk= +github.com/regclient/regclient v0.10.0 h1:V+mbcd/PBwQkd89M2KAPAAC1qMyrw9f5BKvCtPgw/C8= +github.com/regclient/regclient v0.10.0/go.mod h1:fDYvRK2yFUE7pyEXKj+jKzTAtuwXsGQ6Tr0KPvxwP00= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= @@ -480,8 +484,13 @@ github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -496,6 +505,8 @@ github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4d github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= +github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -588,6 +599,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/docs_test.go b/internal/docs_test.go index 099233844..ee9d5cf6c 100644 --- a/internal/docs_test.go +++ b/internal/docs_test.go @@ -22,7 +22,6 @@ package internal import ( "bufio" - "bytes" "fmt" "go/ast" "go/token" @@ -51,6 +50,7 @@ import ( schedulerApi "github.com/arangodb/kube-arangodb/pkg/apis/scheduler/v1beta1" storageApi "github.com/arangodb/kube-arangodb/pkg/apis/storage/v1alpha" "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/helm" ) const ( @@ -58,9 +58,7 @@ const ( apiIndexPageTitle = "CRD reference" ) -func (d DocDefinitions) RenderMarkdown(t *testing.T, repositoryPath string) []byte { - out := bytes.NewBuffer(nil) - +func (d DocDefinitions) RenderMarkdown(t *testing.T, out io.Writer, repositoryPath string) { els := 0 for _, el := range d { @@ -189,8 +187,55 @@ func (d DocDefinitions) RenderMarkdown(t *testing.T, repositoryPath string) []by } } } +} + +func Test_GenerateSecondaryAPIDocs(t *testing.T) { + root := os.Getenv("ROOT") + require.NotEmpty(t, root) + + fset := token.NewFileSet() + + fields := parseSourceFiles(t, root, fset, path.Join(root, "pkg/util/k8sutil/helm")) - return out.Bytes() + out, err := os.OpenFile(path.Join(root, "docs/platform.install.md"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + require.NoError(t, err) + + defer func() { + require.NoError(t, out.Close()) + }() + + writeFrontMatter(t, out, map[string]string{ + "layout": "page", + "title": "Platform Installation File Schema", + "parent": "ArangoDBPlatform", + }) + writef(t, out, "# Installation Definition\n\n") + writef(t, out, "## Example\n\n") + writef(t, out, "```yaml\n") + writef(t, out, `packages: + nginx: # OCI + chart: "oci://ghcr.io/nginx/charts/nginx-ingress:2.3.1" + version: 2.3.1 + prometheus: # Helm Index + chart: "index://prometheus-community.github.io/helm-charts" + version: 1.3.1 + alertmanager: # Remote Chart + chart: "https://github.com/prometheus-community/helm-charts/releases/download/alertmanager-0.1.0/alertmanager-0.1.0.tgz" + version: "0.1.0" + local: # Local File + chart: "file:///tmp/local-0.1.0.tgz" + version: "0.1.0" + inline: # Inline + chart: "" + version: "0.2.5" + platform: # Platform LicenseManager + version: v3.0.11 +`) + writef(t, out, "```\n\n") + + generateDocsOut(t, "Package", map[string]interface{}{ + "Package": helm.Package{}, + }, fields, fset, out) } func Test_GenerateAPIDocs(t *testing.T) { @@ -503,36 +548,41 @@ func prepareGitHubTreePath(t *testing.T, root string) string { return fmt.Sprintf("https://github.com/arangodb/kube-arangodb/blob/%s", ref) } -func generateDocs(t *testing.T, objects map[string]map[string]interface{}, fields map[string]*ast.Field, fs *token.FileSet) map[string]string { +func generateDocsOut(t *testing.T, objectName string, sections map[string]interface{}, fields map[string]*ast.Field, fs *token.FileSet, out io.Writer) { root := os.Getenv("ROOT") require.NotEmpty(t, root) - outPaths := make(map[string]string) - repositoryPath := prepareGitHubTreePath(t, root) - for objectName, sections := range objects { - t.Run(objectName, func(t *testing.T) { - renderSections := map[string][]byte{} - for section, fieldInstance := range sections { - t.Run(section, func(t *testing.T) { + t.Run(objectName, func(t *testing.T) { + for _, section := range util.SortKeys(sections) { + writef(t, out, "## %s\n\n", util.BoolSwitch(section == "", "Object", section)) - sectionParsed := iterateOverObject(t, fields, "", goStrings.ToLower(section), reflect.TypeOf(fieldInstance), "") + t.Run(section, func(t *testing.T) { + fieldInstance := sections[section] - defs := make(DocDefinitions, 0, len(sectionParsed)) - for _, el := range sectionParsed { - defs = append(defs, parseDocDefinition(t, root, cleanPrefixPath(el.K.path), el.K.typ, el.K, el.V, fs)) - } - defs.Sort() + sectionParsed := iterateOverObject(t, fields, "", goStrings.ToLower(section), reflect.TypeOf(fieldInstance), "") - renderSections[section] = defs.RenderMarkdown(t, repositoryPath) - }) - } + defs := make(DocDefinitions, 0, len(sectionParsed)) + for _, el := range sectionParsed { + defs = append(defs, parseDocDefinition(t, root, cleanPrefixPath(el.K.path), el.K.typ, el.K, el.V, fs)) + } + defs.Sort() + + defs.RenderMarkdown(t, out, repositoryPath) + }) + } + }) +} - fileName := fmt.Sprintf("%s.md", objectName) - outPaths[objectName] = fileName +func generateDocs(t *testing.T, objects map[string]map[string]interface{}, fields map[string]*ast.Field, fs *token.FileSet) { + root := os.Getenv("ROOT") + require.NotEmpty(t, root) + + for objectName, sections := range objects { + t.Run(objectName, func(t *testing.T) { outPath := path.Join(root, "docs/api", fmt.Sprintf("%s.md", objectName)) - out, err := os.OpenFile(outPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + out, err := os.OpenFile(outPath, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0644) require.NoError(t, err) defer func() { @@ -540,6 +590,7 @@ func generateDocs(t *testing.T, objects map[string]map[string]interface{}, field }() objName := goStrings.ReplaceAll(objectName, ".", " ") + writeFrontMatter(t, out, map[string]string{ "layout": "page", "title": objName, @@ -547,15 +598,9 @@ func generateDocs(t *testing.T, objects map[string]map[string]interface{}, field }) writef(t, out, "# API Reference for %s\n\n", objName) - util.IterateSorted(renderSections, func(name string, section []byte) { - writef(t, out, "## %s\n\n", util.BoolSwitch(name == "", "Object", name)) - - _, err = out.Write(section) - require.NoError(t, err) - }) + generateDocsOut(t, objectName, sections, fields, fs, out) }) } - return outPaths } func write(t *testing.T, out io.Writer, format string) { diff --git a/pkg/license_manager/client.go b/pkg/license_manager/client.go index 8ac8ee777..10f3d7ced 100644 --- a/pkg/license_manager/client.go +++ b/pkg/license_manager/client.go @@ -74,6 +74,8 @@ func NewClientFromConn(conn driver.Connection) Client { } type Client interface { + Identity(ctx context.Context) (Identity, error) + License(ctx context.Context, req LicenseRequest) (LicenseResponse, error) Registry(ctx context.Context) (RegistryResponse, error) @@ -92,6 +94,9 @@ type LicenseResponse struct { Expires *ugrpc.Object[*timestamppb.Timestamp] `json:"expires,omitempty"` } +type Identity struct { +} + type RegistryResponse struct { Token string `json:"token"` } @@ -100,6 +105,10 @@ type client struct { conn driver.Connection } +func (c client) Identity(ctx context.Context) (Identity, error) { + return arangod.GetRequest[Identity](ctx, c.conn, "_api", "v1", "identity").AcceptCode(200).Response() +} + func (c client) RegistryConfig(ctx context.Context, endpoint, id string, token *string, stages ...Stage) ([]byte, error) { var t string diff --git a/pkg/platform/pack/cache.go b/pkg/platform/pack/cache.go index 38d834f7c..d0d3e42a6 100644 --- a/pkg/platform/pack/cache.go +++ b/pkg/platform/pack/cache.go @@ -29,7 +29,7 @@ import ( "path" "sync" - "github.com/pkg/errors" + "github.com/arangodb/kube-arangodb/pkg/util/errors" ) func NewCache(path string) Cache { diff --git a/pkg/platform/pack/export.go b/pkg/platform/pack/export.go index de64c7048..7125cbab4 100644 --- a/pkg/platform/pack/export.go +++ b/pkg/platform/pack/export.go @@ -30,7 +30,6 @@ import ( "sync" "time" - "github.com/pkg/errors" "github.com/regclient/regclient" "github.com/regclient/regclient/types/descriptor" "github.com/regclient/regclient/types/errs" @@ -41,6 +40,7 @@ import ( "github.com/arangodb/kube-arangodb/pkg/logging" "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/executor" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/helm" ) @@ -122,22 +122,9 @@ func (r *exportPackageSet) exportPackage(name string, spec helm.PackageSpec) exe log.Info("Extracted chart") }() - var chart helm.Chart - - if spec.Chart.IsZero() { - ref, err := ChartReference(r.endpoint, spec.GetStage(), name, spec.Version) - if err != nil { - return err - } - - loadedChart, err := ExportChart(ctx, r.client, ref) - if err != nil { - return err - } - - chart = loadedChart - } else { - chart = helm.Chart(spec.Chart) + chart, err := ResolvePackageSpec(ctx, r.endpoint, name, spec, r.client, nil) + if err != nil { + return err } var chartProto ProtoChart diff --git a/pkg/platform/pack/import.go b/pkg/platform/pack/import.go index a724990fb..6c311efe6 100644 --- a/pkg/platform/pack/import.go +++ b/pkg/platform/pack/import.go @@ -23,7 +23,7 @@ package pack import ( "archive/zip" "context" - "errors" + "encoding/base64" "fmt" "io" "os" @@ -38,6 +38,7 @@ import ( "github.com/arangodb/kube-arangodb/pkg/logging" "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/executor" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/helm" ) @@ -113,7 +114,7 @@ func (i *importPackageSet) run(p Proto) executor.RunFunc { return err } - pkgS.Chart = data + pkgS.Chart = util.NewType(base64.StdEncoding.EncodeToString(data)) pkgS.Version = v.Version var versions ProtoValues @@ -244,6 +245,11 @@ func (i *importPackageSet) importBlob(src ref.Ref, desc descriptor.Descriptor) e return err } + if _, err := i.client.BlobHead(ctx, src, desc); err == nil { + log.Info("Blob exists") + return nil + } + if _, err := i.client.BlobPut(ctx, src, desc, qs); err != nil { return err } diff --git a/pkg/platform/pack/utils.go b/pkg/platform/pack/utils.go index f98355a80..234a6080a 100644 --- a/pkg/platform/pack/utils.go +++ b/pkg/platform/pack/utils.go @@ -22,25 +22,104 @@ package pack import ( "context" + "encoding/base64" "fmt" "io" + goHttp "net/http" + "os" + goStrings "strings" "github.com/regclient/regclient" "github.com/regclient/regclient/types/mediatype" "github.com/regclient/regclient/types/ref" + "github.com/arangodb/kube-arangodb/pkg/util" "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/helm" ) -func ChartReference(endpoint, stage, name, version string) (ref.Ref, error) { - if stage == "prd" { - endpoint = fmt.Sprintf("helm.%s", endpoint) - } else { - endpoint = fmt.Sprintf("%s.helm.%s", stage, endpoint) +func ResolvePackageSpec(ctx context.Context, endpoint, name string, in helm.PackageSpec, reg *regclient.RegClient, chttp *goHttp.Client) (helm.Chart, error) { + if err := in.Validate(); err != nil { + return helm.Chart{}, err } - return ref.New(fmt.Sprintf("%s/%s:%s", endpoint, name, version)) + switch in.PackageType() { + case helm.PackageTypePlatform: + if in.GetStage() == "prd" { + endpoint = fmt.Sprintf("helm.%s", endpoint) + } else { + endpoint = fmt.Sprintf("%s.helm.%s", in.GetStage(), endpoint) + } + + r, err := ref.New(fmt.Sprintf("%s/%s:%s", endpoint, name, in.Version)) + if err != nil { + return helm.Chart{}, err + } + + return ExportChart(ctx, reg, r) + case helm.PackageTypeInline: + return base64.StdEncoding.DecodeString(util.WithDefault(in.Chart)) + case helm.PackageTypeRemote: + e := util.WithDefault(in.Chart) + + if chttp == nil { + chttp = goHttp.DefaultClient + } + + req, err := goHttp.NewRequestWithContext(ctx, "GET", e, nil) + if err != nil { + return helm.Chart{}, err + } + + resp, err := chttp.Do(req) + if err != nil { + return helm.Chart{}, err + } + + defer resp.Body.Close() + + return io.ReadAll(resp.Body) + + case helm.PackageTypeFile: + e := util.WithDefault(in.Chart) + + e = goStrings.TrimPrefix(e, "file://") + + return os.ReadFile(e) + + case helm.PackageTypeOCI: + e := util.WithDefault(in.Chart) + e = goStrings.TrimPrefix(e, "oci://") + + r, err := ref.New(e) + if err != nil { + return helm.Chart{}, err + } + + return ExportChart(ctx, reg, r) + case helm.PackageTypeIndex: + e := util.WithDefault(in.Chart) + e = goStrings.TrimPrefix(e, "index://") + + hm, err := helm.NewChartManager(ctx, chttp, "https://%s/index.yaml", e) + if err != nil { + return helm.Chart{}, err + } + + p, ok := hm.Get(name) + if !ok { + return nil, errors.Errorf("Package %s not found", name) + } + + v, ok := p.Get(in.Version) + if !ok { + return nil, errors.Errorf("Package %s version %s not found", name, in.Version) + } + + return v.Get(ctx) + default: + return helm.Chart{}, fmt.Errorf("invalid package type") + } } func ExportChart(ctx context.Context, client *regclient.RegClient, src ref.Ref) (helm.Chart, error) { diff --git a/pkg/platform/package_install.go b/pkg/platform/package_install.go index f0b6248cb..8dd15b81a 100644 --- a/pkg/platform/package_install.go +++ b/pkg/platform/package_install.go @@ -246,12 +246,7 @@ func packageInstallRunInstallChart(cmd *cobra.Command, h executor.Handler, clien log.Info("Calculating installation") - ref, err := pack.ChartReference(endpoint, packageSpec.GetStage(), name, packageSpec.Version) - if err != nil { - return err - } - - chart, err := pack.ExportChart(cmd.Context(), reg, ref) + chart, err := pack.ResolvePackageSpec(ctx, endpoint, name, packageSpec, reg, nil) if err != nil { return err } diff --git a/pkg/platform/package_test.go b/pkg/platform/package_test.go index 3408c8e57..d91c4d339 100644 --- a/pkg/platform/package_test.go +++ b/pkg/platform/package_test.go @@ -23,6 +23,7 @@ package platform import ( "bytes" _ "embed" + "encoding/base64" "fmt" "io" "os" @@ -238,7 +239,7 @@ func Test_Package(t *testing.T) { Packages: map[string]helm.PackageSpec{ "sample": { Version: "1.0.0", - Chart: sample_1_0_0, + Chart: util.NewType(base64.StdEncoding.EncodeToString(sample_1_0_0)), }, }, }) @@ -249,7 +250,7 @@ func Test_Package(t *testing.T) { Packages: map[string]helm.PackageSpec{ "sample": { Version: "1.0.0", - Chart: sample_1_0_0, + Chart: util.NewType(base64.StdEncoding.EncodeToString(sample_1_0_0)), }, }, Releases: map[string]helm.PackageRelease{ @@ -281,7 +282,7 @@ func Test_Package(t *testing.T) { Packages: map[string]helm.PackageSpec{ "sample": { Version: "1.0.0", - Chart: sample_1_0_0, + Chart: util.NewType(base64.StdEncoding.EncodeToString(sample_1_0_0)), }, }, Releases: map[string]helm.PackageRelease{ @@ -331,7 +332,7 @@ func Test_Package(t *testing.T) { Packages: map[string]helm.PackageSpec{ "sample": { Version: "1.0.0", - Chart: sample_1_0_0, + Chart: util.NewType(base64.StdEncoding.EncodeToString(sample_1_0_0)), Overrides: helm.Values(sharedApi.NewAnyT(t, map[string]string{ "data": "Ov2", })), diff --git a/pkg/util/cli/flag.go b/pkg/util/cli/flag.go index 9671b5220..1d3e5ff61 100644 --- a/pkg/util/cli/flag.go +++ b/pkg/util/cli/flag.go @@ -85,20 +85,31 @@ func (f Flag[T]) GetName() string { } func (f Flag[T]) Validate(cmd *cobra.Command) error { + if _, err := f.Get(cmd); err != nil { + return err + } + + return nil +} + +func (f Flag[T]) Get(cmd *cobra.Command) (T, error) { if cmd.Flags().Lookup(f.Name) == nil { - return nil + return util.Default[T](), nil } - v, err := f.Get(cmd) + v, err := f.get(cmd) if err != nil { - return err + return util.Default[T](), err } if f.Check != nil { - return f.Check(v) + if err := f.Check(v); err != nil { + return util.Default[T](), err + } + return v, nil } - return nil + return v, nil } func (f Flag[T]) Register(cmd *cobra.Command) error { @@ -170,7 +181,7 @@ func (f Flag[T]) Register(cmd *cobra.Command) error { return nil } -func (f Flag[T]) Get(cmd *cobra.Command) (T, error) { +func (f Flag[T]) get(cmd *cobra.Command) (T, error) { switch util.TypeOf[T]() { case util.TypeOf[string](): v, err := cmd.Flags().GetString(f.Name) diff --git a/pkg/util/cli/lm.go b/pkg/util/cli/lm.go index 995ce379e..ba9aec56d 100644 --- a/pkg/util/cli/lm.go +++ b/pkg/util/cli/lm.go @@ -196,7 +196,19 @@ func (l licenseManager) Client(cmd *cobra.Command) (lmanager.Client, error) { return nil, err } - return lmanager.NewClient(endpoint, cid, cs) + c, err := lmanager.NewClient(endpoint, cid, cs) + if err != nil { + return nil, err + } + + id, err := c.Identity(cmd.Context()) + if err != nil { + return nil, err + } + + logger.JSON("identity", id).Info("Using identity for client") + + return c, nil } func (l licenseManager) Register(cmd *cobra.Command) error { @@ -210,6 +222,5 @@ func (l licenseManager) Register(cmd *cobra.Command) error { func (l licenseManager) Validate(cmd *cobra.Command) error { return ValidateFlags( l.endpoint, - l.client, )(cmd, nil) } diff --git a/pkg/util/cli/registry.go b/pkg/util/cli/registry.go index 1dea524e9..72040a1c6 100644 --- a/pkg/util/cli/registry.go +++ b/pkg/util/cli/registry.go @@ -21,7 +21,6 @@ package cli import ( - "log/slog" goHttp "net/http" "github.com/regclient/regclient" @@ -89,11 +88,9 @@ func (r registry) Validate(cmd *cobra.Command) error { func (r registry) Client(cmd *cobra.Command, lm LicenseManager) (*regclient.RegClient, error) { var flags = make([]regclient.Opt, 0, 3) - slog.SetLogLoggerLevel(slog.LevelDebug) - flags = append(flags, regclient.WithConfigHostDefault(config.Host{ ReqConcurrent: 8, - }), regclient.WithSlog(slog.Default())) + })) flags = append(flags, regclient.WithRegOpts(reg.WithTransport(&goHttp.Transport{ MaxConnsPerHost: 64, diff --git a/pkg/util/k8sutil/helm/package.go b/pkg/util/k8sutil/helm/package.go index 06904046d..e077adec0 100644 --- a/pkg/util/k8sutil/helm/package.go +++ b/pkg/util/k8sutil/helm/package.go @@ -22,37 +22,136 @@ package helm import ( "context" + "encoding/base64" + goStrings "strings" "helm.sh/helm/v3/pkg/action" meta "k8s.io/apimachinery/pkg/apis/meta/v1" platformApi "github.com/arangodb/kube-arangodb/pkg/apis/platform/v1beta1" - sharedApi "github.com/arangodb/kube-arangodb/pkg/apis/shared/v1" + shared "github.com/arangodb/kube-arangodb/pkg/apis/shared" + "github.com/arangodb/kube-arangodb/pkg/util" utilConstants "github.com/arangodb/kube-arangodb/pkg/util/constants" "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/kclient" ) +type PackageType int + +const ( + PackageTypePlatform PackageType = iota + PackageTypeFile + PackageTypeRemote + PackageTypeOCI + PackageTypeInline + PackageTypeIndex +) + type Package struct { + // Packages keeps the map of Packages to be installed Packages map[string]PackageSpec `json:"packages,omitempty"` + // Releases keeps the map of Releases to be installed Releases map[string]PackageRelease `json:"releases,omitempty"` } func (pkg *Package) Validate() error { - return nil + if pkg == nil { + return nil + } + return errors.Errors( + shared.PrefixResourceErrors("packages", shared.ValidateMap(pkg.Packages, func(s string, spec PackageSpec) error { + return spec.Validate() + })), + shared.PrefixResourceErrors("releases", shared.ValidateMap(pkg.Releases, func(s string, spec PackageRelease) error { + return spec.Validate() + })), + ) } type PackageSpec struct { + // Stage defines stage used in the fetch from LicenseManager Stage *string `json:"stage,omitempty"` + // Version keeps the version of the PackageSpec Version string `json:"version"` - Chart sharedApi.Data `json:"chart,omitempty"` - + // Chart defines override of the PackageSpec + // It supports multiple modes: + // - If undefined, LicenseManager OCI Repository is used + // - If starts with `file://` chart is fetched from local FileSystem + // - If starts with `http://` or `https://` chart is fetched from the remote URL + // - If starts with `index://` chart is fetched using Helm YAML Index File stricture (using version and name) + // - If Starts with `oci://` chart is fetched from Registry Compatible OCI Repository + // - If none above match, chart is decoded using Base64 encoding + Chart *string `json:"chart,omitempty"` + + // Overrides defines Values to override the Helm Chart Defaults (merged with Service Overrides) Overrides Values `json:"overrides,omitempty"` } +func (p PackageSpec) PackageType() PackageType { + if c := p.Chart; c != nil { + // File + if goStrings.HasPrefix(*c, "file://") { + return PackageTypeFile + } + + // HTTP + if goStrings.HasPrefix(*c, "https://") || goStrings.HasPrefix(*c, "http://") { + return PackageTypeRemote + } + + // OCI + if goStrings.HasPrefix(*c, "oci://") { + return PackageTypeOCI + } + + // Helm Index File + if goStrings.HasPrefix(*c, "index://") { + return PackageTypeIndex + } + + return PackageTypeInline + } + return PackageTypePlatform +} + +func (p PackageSpec) Validate() error { + if c := p.Chart; c != nil { + // File + if goStrings.HasPrefix(*c, "file://") { + return nil + } + + // HTTP + if goStrings.HasPrefix(*c, "https://") || goStrings.HasPrefix(*c, "http://") { + return nil + } + + // OCI + if goStrings.HasPrefix(*c, "oci://") { + return nil + } + + // Helm Index File + if goStrings.HasPrefix(*c, "index://") { + return nil + } + + // Base64 + if _, err := base64.StdEncoding.DecodeString(*c); err != nil { + return errors.Wrapf(err, "Unable to decode chart data") + } + } else { + if p.Version == "" { + return errors.Errorf("Version is required if chart is not provided") + } + } + + return nil +} + func (p PackageSpec) GetStage() string { if p.Stage == nil { return "prd" @@ -62,11 +161,18 @@ func (p PackageSpec) GetStage() string { } type PackageRelease struct { + // Package keeps the name of the Chart used from the installation script. + // References to value provided in Packages Package string `json:"package"` + // Overrides defines Values to override the Helm Chart Defaults during installation Overrides Values `json:"overrides,omitempty"` } +func (p PackageRelease) Validate() error { + return nil +} + func NewPackage(ctx context.Context, client kclient.Client, namespace, deployment string) (*Package, error) { hclient, err := NewClient(Configuration{ Namespace: namespace, @@ -98,7 +204,7 @@ func NewPackage(ctx context.Context, client kclient.Client, namespace, deploymen out.Packages[name] = PackageSpec{ Version: det.GetVersion(), Overrides: Values(info.Overrides), - Chart: c.Status.Info.Definition, + Chart: util.NewType(base64.StdEncoding.EncodeToString(c.Status.Info.Definition)), } } } From 4bbaf4e21da628dee374c701b00f6d08ccfe5e38 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Tue, 18 Nov 2025 13:46:39 +0000 Subject: [PATCH 2/3] Iter --- docs/platform.install.md | 2 +- pkg/util/k8sutil/helm/package.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/platform.install.md b/docs/platform.install.md index 61c5dfeec..68dd6879b 100644 --- a/docs/platform.install.md +++ b/docs/platform.install.md @@ -40,7 +40,7 @@ It supports multiple modes: - If undefined, LicenseManager OCI Repository is used - If starts with `file://` chart is fetched from local FileSystem - If starts with `http://` or `https://` chart is fetched from the remote URL -- If starts with `index://` chart is fetched using Helm YAML Index File stricture (using version and name) +- If starts with `index://` chart is fetched using Helm YAML Index File structure (using version and name) - If Starts with `oci://` chart is fetched from Registry Compatible OCI Repository - If none above match, chart is decoded using Base64 encoding diff --git a/pkg/util/k8sutil/helm/package.go b/pkg/util/k8sutil/helm/package.go index e077adec0..3b32be212 100644 --- a/pkg/util/k8sutil/helm/package.go +++ b/pkg/util/k8sutil/helm/package.go @@ -81,7 +81,7 @@ type PackageSpec struct { // - If undefined, LicenseManager OCI Repository is used // - If starts with `file://` chart is fetched from local FileSystem // - If starts with `http://` or `https://` chart is fetched from the remote URL - // - If starts with `index://` chart is fetched using Helm YAML Index File stricture (using version and name) + // - If starts with `index://` chart is fetched using Helm YAML Index File structure (using version and name) // - If Starts with `oci://` chart is fetched from Registry Compatible OCI Repository // - If none above match, chart is decoded using Base64 encoding Chart *string `json:"chart,omitempty"` From 6d24547fe37a3f4a58c56a6302faee62a000cfa8 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Tue, 18 Nov 2025 13:53:26 +0000 Subject: [PATCH 3/3] Iter --- docs/platform.install.md | 6 +++--- pkg/util/k8sutil/helm/package.go | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/platform.install.md b/docs/platform.install.md index 68dd6879b..84cfab6ac 100644 --- a/docs/platform.install.md +++ b/docs/platform.install.md @@ -48,7 +48,7 @@ It supports multiple modes: ### .package.packages.\.overrides -Type: `array` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.3.1/pkg/util/k8sutil/helm/package.go#L90) +Type: `Object` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.3.1/pkg/util/k8sutil/helm/package.go#L91) Overrides defines Values to override the Helm Chart Defaults (merged with Service Overrides) @@ -72,7 +72,7 @@ Version keeps the version of the PackageSpec ### .package.releases.\.overrides -Type: `array` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.3.1/pkg/util/k8sutil/helm/package.go#L169) +Type: `Object` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.3.1/pkg/util/k8sutil/helm/package.go#L171) Overrides defines Values to override the Helm Chart Defaults during installation @@ -80,7 +80,7 @@ Overrides defines Values to override the Helm Chart Defaults during installation ### .package.releases.\.package -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.3.1/pkg/util/k8sutil/helm/package.go#L166) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.3.1/pkg/util/k8sutil/helm/package.go#L167) Package keeps the name of the Chart used from the installation script. References to value provided in Packages diff --git a/pkg/util/k8sutil/helm/package.go b/pkg/util/k8sutil/helm/package.go index 3b32be212..dc31e1471 100644 --- a/pkg/util/k8sutil/helm/package.go +++ b/pkg/util/k8sutil/helm/package.go @@ -87,6 +87,7 @@ type PackageSpec struct { Chart *string `json:"chart,omitempty"` // Overrides defines Values to override the Helm Chart Defaults (merged with Service Overrides) + // +doc/type: Object Overrides Values `json:"overrides,omitempty"` } @@ -166,6 +167,7 @@ type PackageRelease struct { Package string `json:"package"` // Overrides defines Values to override the Helm Chart Defaults during installation + // +doc/type: Object Overrides Values `json:"overrides,omitempty"` }