diff --git a/internal/deployment-repo/deploymentRepoManager.go b/internal/deployment-repo/deploymentRepoManager.go index e5a777c..7caee4a 100644 --- a/internal/deployment-repo/deploymentRepoManager.go +++ b/internal/deployment-repo/deploymentRepoManager.go @@ -270,7 +270,12 @@ func (m *DeploymentRepoManager) ApplyTemplates(ctx context.Context) error { return fmt.Errorf("failed to apply fluxcd image automation controller template input: %w", err) } - err = TemplateDir(m.templatesDir, templateInput, m.gitRepo) + err = util.CopyDir(m.ExtraManifestDir, filepath.Join(m.templatesDir, ResourcesDirectoryName, OpenMCPDirectoryName, ExtraManifestsDirectory)) + if err != nil { + return fmt.Errorf("failed to copy extra manifests from %s to deployment repository: %w", m.ExtraManifestDir, err) + } + + err = TemplateDir(ctx, m.templatesDir, templateInput, m.compGetter, m.gitRepo) if err != nil { return fmt.Errorf("failed to apply templates from directory %s: %w", m.templatesDir, err) } @@ -466,18 +471,6 @@ func (m *DeploymentRepoManager) ApplyExtraManifests(_ context.Context) error { // if an extra manifest directory is specified, copy its contents to the deployment repository logger.Infof("Applying extra manifests from %s to deployment repository", m.ExtraManifestDir) - err := util.CopyDir(m.ExtraManifestDir, filepath.Join(m.gitRepoDir, ResourcesDirectoryName, OpenMCPDirectoryName, ExtraManifestsDirectory)) - if err != nil { - return fmt.Errorf("failed to copy extra manifests from %s to deployment repository: %w", m.ExtraManifestDir, err) - } - workTree, err := m.gitRepo.Worktree() - if err != nil { - return fmt.Errorf("failed to get worktree: %w", err) - } - _, err = workTree.Add(filepath.Join(ResourcesDirectoryName, OpenMCPDirectoryName, ExtraManifestsDirectory)) - if err != nil { - return fmt.Errorf("failed to add extra manifests to git index: %w", err) - } entries, err := os.ReadDir(m.ExtraManifestDir) if err != nil { @@ -499,6 +492,7 @@ func (m *DeploymentRepoManager) ApplyExtraManifests(_ context.Context) error { return nil } +// UpdateResourcesKustomization updates the resources kustomization file in the deployment repository to include all applied resources. func (m *DeploymentRepoManager) UpdateResourcesKustomization() error { logger := log.GetLogger() files := make([]string, 0, diff --git a/internal/deployment-repo/deploymentRepoManager_test.go b/internal/deployment-repo/deploymentRepoManager_test.go index 3bc3b84..ee52092 100644 --- a/internal/deployment-repo/deploymentRepoManager_test.go +++ b/internal/deployment-repo/deploymentRepoManager_test.go @@ -103,6 +103,9 @@ func TestDeploymentRepoManager(t *testing.T) { err = deploymentRepoManager.ApplyCustomResourceDefinitions(t.Context()) assert.NoError(t, err) + err = deploymentRepoManager.ApplyExtraManifests(t.Context()) + assert.NoError(t, err) + err = deploymentRepoManager.UpdateResourcesKustomization() assert.NoError(t, err) @@ -119,7 +122,7 @@ func TestDeploymentRepoManager(t *testing.T) { expectedRepoDir := "./testdata/01/expected-repo" actualRepoDir := originDir - testutils.AssertDirectoriesEqualWithNormalization(t, expectedRepoDir, actualRepoDir, createGitRepoNormalizer(originDir)) + testutils.AssertDirectoriesEqualWithNormalization(t, expectedRepoDir, actualRepoDir, createTestNormalizer(originDir, ctfIn)) fluxKustomization := &unstructured.Unstructured{} fluxKustomization.SetGroupVersionKind(schema.GroupVersionKind{ @@ -132,12 +135,16 @@ func TestDeploymentRepoManager(t *testing.T) { assert.NoError(t, err) } -// CreateGitRepoNormalizer creates a normalizer function that replaces dynamic git repository URLs -func createGitRepoNormalizer(actualRepoURL string) func(string, string) string { +// createTestNormalizer returns a function that normalizes file content by replacing actual repository URLs with placeholders. +func createTestNormalizer(actualRepoURL, actualOCMRepoURL string) func(string, string) string { return func(content, filePath string) string { // For gitrepo.yaml files, replace the actual repo URL with a placeholder if strings.Contains(filePath, "gitrepo.yaml") { - return strings.ReplaceAll(content, actualRepoURL, "{{GIT_REPO_URL}}") + content = strings.ReplaceAll(content, actualRepoURL, "{{GIT_REPO_URL}}") + } + // For files that may contain OCM repository URLs, replace the actual repo URL with a placeholder + if strings.Contains(filePath, ".yaml") || strings.Contains(filePath, ".yml") || strings.Contains(filePath, ".json") { + content = strings.ReplaceAll(content, actualOCMRepoURL, "{{OCM_REPO_URL}}") } return content } diff --git a/internal/deployment-repo/templater.go b/internal/deployment-repo/templater.go index 4e843f0..39c7520 100644 --- a/internal/deployment-repo/templater.go +++ b/internal/deployment-repo/templater.go @@ -20,7 +20,7 @@ import ( // TemplateDir processes the template files in the specified directory and writes // the rendered content to the corresponding files in the Git repository's worktree. // It uses the provided template directory and Git repository to perform the operations. -func TemplateDir(templateDirectory string, templateInput map[string]interface{}, repo *git.Repository) error { +func TemplateDir(ctx context.Context, templateDirectory string, templateInput map[string]interface{}, compGetter *ocmcli.ComponentGetter, repo *git.Repository) error { logger := log.GetLogger() workTree, err := repo.Worktree() @@ -38,7 +38,7 @@ func TemplateDir(templateDirectory string, templateInput map[string]interface{}, } }() - te := template.NewTemplateExecution().WithMissingKeyOption("zero") + te := template.NewTemplateExecution().WithOCMComponentGetter(ctx, compGetter).WithMissingKeyOption("zero") // Recursively walk through all files in the template directory err = filepath.WalkDir(templateDirectory, func(path string, d os.DirEntry, walkError error) error { diff --git a/internal/deployment-repo/testdata/01/component-constructor.yaml b/internal/deployment-repo/testdata/01/component-constructor.yaml index a05690c..7a083e7 100644 --- a/internal/deployment-repo/testdata/01/component-constructor.yaml +++ b/internal/deployment-repo/testdata/01/component-constructor.yaml @@ -25,6 +25,10 @@ components: name: gitops-templates version: v0.1.1 + - componentName: github.com/openmcp-project/openmcp/releasechannel + name: releasechannel + version: v2.1.3 + resources: - name: fluxcd-source-controller version: v1.6.2 @@ -152,4 +156,44 @@ components: version: v0.1.1 input: type: dir - path: ./templates/openmcp \ No newline at end of file + path: ./templates/openmcp + + - name: github.com/openmcp-project/openmcp/releasechannel + version: v2.1.3 + provider: + name: openmcp-project + + componentReferences: + - componentName: github.com/openmcp-project/openmcp/releasechannel/crossplane + name: crossplane + version: v0.0.1 + + - componentName: github.com/openmcp-project/openmcp/releasechannel/crossplane + name: crossplane + version: v0.0.2 + + - name: github.com/openmcp-project/openmcp/releasechannel/crossplane + version: v0.0.1 + provider: + name: openmcp-project + + resources: + - name: image-crossplane + type: ociImage + version: v0.0.2 + access: + type: ociArtifact + imageReference: ghcr.io/openmcp-project/releasechannel/crossplane:v0.0.2 + + - name: github.com/openmcp-project/openmcp/releasechannel/crossplane + version: v0.0.2 + provider: + name: openmcp-project + + resources: + - name: image-crossplane + type: ociImage + version: v0.0.2 + access: + type: ociArtifact + imageReference: ghcr.io/openmcp-project/releasechannel/crossplane:v0.0.2 \ No newline at end of file diff --git a/internal/deployment-repo/testdata/01/expected-repo/resources/openmcp/extra/test-configmap.yaml b/internal/deployment-repo/testdata/01/expected-repo/resources/openmcp/extra/test-configmap.yaml new file mode 100644 index 0000000..1bbea97 --- /dev/null +++ b/internal/deployment-repo/testdata/01/expected-repo/resources/openmcp/extra/test-configmap.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-configmap + namespace: default +data: + key: "value" + repository: "{{OCM_REPO_URL}}" + crossplaneComponentName: "github.com/openmcp-project/openmcp/releasechannel/crossplane" + crossplaneComponentVersion: "v0.0.1" + crossplaneImage: "ghcr.io/openmcp-project/releasechannel/crossplane" + crossplaneVersions: | + - v0.0.1 + - v0.0.2 \ No newline at end of file diff --git a/internal/deployment-repo/testdata/01/expected-repo/resources/openmcp/kustomization.yaml b/internal/deployment-repo/testdata/01/expected-repo/resources/openmcp/kustomization.yaml index 5d29fe5..d8aee6e 100644 --- a/internal/deployment-repo/testdata/01/expected-repo/resources/openmcp/kustomization.yaml +++ b/internal/deployment-repo/testdata/01/expected-repo/resources/openmcp/kustomization.yaml @@ -23,3 +23,4 @@ resources: - cluster-providers/test.yaml - service-providers/test.yaml - platform-services/test.yaml +- extra/test-configmap.yaml diff --git a/internal/deployment-repo/testdata/01/extra-manifests/test-configmap.yaml b/internal/deployment-repo/testdata/01/extra-manifests/test-configmap.yaml index c6494a3..f8ef894 100644 --- a/internal/deployment-repo/testdata/01/extra-manifests/test-configmap.yaml +++ b/internal/deployment-repo/testdata/01/extra-manifests/test-configmap.yaml @@ -4,4 +4,17 @@ metadata: name: test-configmap namespace: default data: - key: "value" \ No newline at end of file + key: "value" + repository: "{{ getOCMRepository }}" + {{- $releaseChannelCv := getComponentVersionByReference "releasechannel" }} + {{- $crossplaneCv := getComponentVersionByReference $releaseChannelCv "crossplane" }} + {{- $crossplaneImageResource := getResourceFromComponentVersion $crossplaneCv "image-crossplane" }} + {{- $imageReference := dig "access" "imageReference" "none" $crossplaneImageResource }} + {{- $parsedImage := parseImage $imageReference }} + {{- $crossplaneCvMap := componentVersionAsMap $crossplaneCv }} + crossplaneComponentName: "{{ dig "component" "name" "none" $crossplaneCvMap }}" + crossplaneComponentVersion: "{{ dig "component" "version" "none" $crossplaneCvMap }}" + crossplaneImage: "{{ dig "image" "none" $parsedImage }}" + crossplaneVersions: | + {{- $crossplaneVersions := listComponentVersions $crossplaneCv }} +{{ toYaml (sortAlpha $crossplaneVersions) | indent 6 }} \ No newline at end of file diff --git a/internal/ocm-cli/component_getter.go b/internal/ocm-cli/component_getter.go index df6ec74..d82a371 100644 --- a/internal/ocm-cli/component_getter.go +++ b/internal/ocm-cli/component_getter.go @@ -77,6 +77,14 @@ func (g *ComponentGetter) TemplatesResourceName() string { return g.templatesResourceName } +func (g *ComponentGetter) Repository() string { + return g.repo +} + +func (g *ComponentGetter) OCMConfig() string { + return g.ocmConfig +} + func (g *ComponentGetter) GetReferencedComponentVersion(ctx context.Context, parentCV *ComponentVersion, refName string) (*ComponentVersion, error) { ref, err := parentCV.GetComponentReference(refName) if err != nil { diff --git a/internal/ocm-cli/ocm.go b/internal/ocm-cli/ocm.go index c75bf8e..988dbed 100644 --- a/internal/ocm-cli/ocm.go +++ b/internal/ocm-cli/ocm.go @@ -3,10 +3,12 @@ package ocm_cli import ( "context" "fmt" + "io" "os" "os/exec" "strings" + yaml2 "k8s.io/apimachinery/pkg/util/yaml" "sigs.k8s.io/yaml" ) @@ -46,16 +48,40 @@ func Execute(ctx context.Context, commands []string, args []string, ocmConfig st return fmt.Errorf("error waiting for ocm command to finish: %w", err) } - // get exit code - if exitError, ok := cmd.ProcessState.Sys().(interface{ ExitStatus() int }); ok { - if exitCode := exitError.ExitStatus(); exitCode != 0 { - return fmt.Errorf("ocm command exited with code %d", exitCode) - } + if cmd.ProcessState.ExitCode() != 0 { + return fmt.Errorf("ocm command exited with code %d", cmd.ProcessState.ExitCode()) } return nil } +func ExecuteOutput(ctx context.Context, commands []string, args []string, ocmConfig string) ([]byte, error) { + var ocmArgs []string + + if ocmConfig != NoOcmConfig { + ocmArgs = append(ocmArgs, "--config", ocmConfig) + + if err := verifyOCMConfig(ocmConfig); err != nil { + return nil, fmt.Errorf("invalid OCM configuration: %w", err) + } + } + + ocmArgs = append(ocmArgs, commands...) + ocmArgs = append(ocmArgs, args...) + + cmd := exec.CommandContext(ctx, "ocm", ocmArgs...) + out, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("error executing ocm command: %w, %q", err, out) + } + + if cmd.ProcessState.ExitCode() != 0 { + return nil, fmt.Errorf("ocm command exited with code %d: %q", cmd.ProcessState.ExitCode(), out) + } + + return out, nil +} + // ComponentVersion represents a version of an OCM component. type ComponentVersion struct { // Component is the OCM component associated with this version. @@ -109,6 +135,11 @@ type Access struct { MediaType *string `json:"mediaType"` } +type ComponentListEntry struct { + Name string `json:"name"` + Version string `json:"version"` +} + var ( OCIImageResourceType = "ociImage" ) @@ -143,24 +174,36 @@ func (cv *ComponentVersion) GetComponentReference(name string) (*ComponentRefere return nil, fmt.Errorf("component reference %s not found in component version %s", name, cv.Component.Name) } -// GetComponentVersion retrieves a component version by its reference using the OCM CLI. -func GetComponentVersion(ctx context.Context, componentReference string, ocmConfig string) (*ComponentVersion, error) { - var ocmArgs []string +func (cv *ComponentVersion) ListComponentVersions(ctx context.Context, ocmConfig string) ([]string, error) { - if ocmConfig != NoOcmConfig { - ocmArgs = append(ocmArgs, "--config", ocmConfig) + out, err := ExecuteOutput(ctx, []string{"list", "componentversion", cv.Repository + "//" + cv.Component.Name}, []string{"--output", "yaml"}, NoOcmConfig) + if err != nil { + return nil, err + } - if err := verifyOCMConfig(ocmConfig); err != nil { - return nil, fmt.Errorf("invalid OCM configuration: %w", err) + cvList := make([]string, 0) + decoder := yaml2.NewYAMLOrJSONDecoder(strings.NewReader(string(out)), 1024) + for { + var entry ComponentListEntry + err = decoder.Decode(&entry) + if err == io.EOF { + break } + if err != nil { + return nil, fmt.Errorf("error decoding component version list: %w", err) + } + cvList = append(cvList, entry.Version) } - ocmArgs = append(ocmArgs, "get", "componentversion", "--output", "yaml", componentReference) + return cvList, nil - cmd := exec.CommandContext(ctx, "ocm", ocmArgs...) - out, err := cmd.CombinedOutput() +} + +// GetComponentVersion retrieves a component version by its reference using the OCM CLI. +func GetComponentVersion(ctx context.Context, componentReference string, ocmConfig string) (*ComponentVersion, error) { + out, err := ExecuteOutput(ctx, []string{"get", "componentversion", componentReference}, []string{"--output", "yaml"}, ocmConfig) if err != nil { - return nil, fmt.Errorf("error executing ocm command: %w, %q", err, out) + return nil, err } var cv ComponentVersion diff --git a/internal/template/template.go b/internal/template/template.go index b2de7f4..500abea 100644 --- a/internal/template/template.go +++ b/internal/template/template.go @@ -2,15 +2,17 @@ package template import ( "bytes" + "context" "strings" gotmpl "text/template" "github.com/Masterminds/sprig/v3" "sigs.k8s.io/yaml" + + ocmcli "github.com/openmcp-project/bootstrapper/internal/ocm-cli" + "github.com/openmcp-project/bootstrapper/internal/util" ) -// toYAML takes an interface, marshals it to yaml, and returns a string. It will -// always return a string, even on marshal error (empty string). func toYAML(v interface{}) string { data, err := yaml.Marshal(v) if err != nil { @@ -20,13 +22,114 @@ func toYAML(v interface{}) string { return strings.TrimSuffix(string(data), "\n") } -// fromYAML takes a string, unmarshals it from yaml, and returns an interface. func fromYAML(input string) (any, error) { var output any err := yaml.Unmarshal([]byte(input), &output) return output, err } +func getComponentVersionByReference(ctx context.Context, compGetter *ocmcli.ComponentGetter, args ...interface{}) *ocmcli.ComponentVersion { + if compGetter == nil { + panic("ComponentGetter must not be nil") + } + + if len(args) < 1 { + panic("at least 1 argument is expected") + } + + var err error + parentCv := compGetter.RootComponentVersion() + referenceName := args[len(args)-1].(string) + + if len(args) == 2 { + parentCv = args[0].(*ocmcli.ComponentVersion) + } + + cv, err := compGetter.GetReferencedComponentVersionRecursive(ctx, parentCv, referenceName) + if err != nil || cv == nil { + return nil + } + + return cv +} + +func componentVersionAsMap(cv *ocmcli.ComponentVersion) map[string]interface{} { + if cv == nil { + return nil + } + + m, err := yaml.Marshal(cv) + if err != nil { + return nil + } + + var output map[string]interface{} + err = yaml.Unmarshal(m, &output) + if err != nil { + return nil + } + + return output +} + +func getResourceFromComponentVersion(compGetter *ocmcli.ComponentGetter, cv *ocmcli.ComponentVersion, resourceName string) map[string]interface{} { + if compGetter == nil { + panic("ComponentGetter must not be nil") + } + + res, err := cv.GetResource(resourceName) + if err != nil || res == nil { + return nil + } + + m, err := yaml.Marshal(res) + if err != nil { + return nil + } + + var output map[string]interface{} + err = yaml.Unmarshal(m, &output) + if err != nil { + return nil + } + + return output +} + +func getOCMRepository(compGetter *ocmcli.ComponentGetter) string { + if compGetter == nil { + panic("ComponentGetter must not be nil") + } + + return compGetter.Repository() +} + +func listComponentVersions(ctx context.Context, compGetter *ocmcli.ComponentGetter, cv *ocmcli.ComponentVersion) []string { + if compGetter == nil { + panic("ComponentGetter must not be nil") + } + + versions, err := cv.ListComponentVersions(ctx, compGetter.OCMConfig()) + if err != nil { + return nil + } + + return versions +} + +func parseImageReference(imageRef string) map[string]interface{} { + imageName, tag, digest, err := util.ParseImageVersionAndTag(imageRef) + if err != nil { + return nil + } + + return map[string]interface{}{ + "image": imageName, + "tag": tag, + "digest": digest, + } +} + // TemplateExecution is a struct that provides methods to execute templates with input data. type TemplateExecution struct { funcMaps []gotmpl.FuncMap @@ -44,8 +147,16 @@ func NewTemplateExecution() *TemplateExecution { t.funcMaps = append(t.funcMaps, sprig.FuncMap()) t.funcMaps = append(t.funcMaps, gotmpl.FuncMap{ - "toYaml": toYAML, + // toYaml takes an interface, marshals it to yaml, and returns a string. It will + // always return a string, even on marshal error (empty string). + "toYaml": toYAML, + // fromYaml takes a string, unmarshals it from yaml, and returns an interface. + // It returns an error if the unmarshal fails. "fromYaml": fromYAML, + // parseImage takes a container image string and returns a map with the keys "name", "tag", and "digest". + // If no tag is specified, it defaults to "latest". If a digest is present, it is returned as well. + // If no digest is present, the "digest" key will have an empty string as value. + "parseImage": parseImageReference, }) return t } @@ -74,6 +185,53 @@ func (t *TemplateExecution) WithMissingKeyOption(option string) *TemplateExecuti return t } +func (t *TemplateExecution) WithOCMComponentGetter(ctx context.Context, compGetter *ocmcli.ComponentGetter) *TemplateExecution { + if compGetter != nil { + t.funcMaps = append(t.funcMaps, gotmpl.FuncMap{ + // getOCMRepository returns the OCM repository URL from the ComponentGetter. + // If the ComponentGetter is nil, it panics. + "getOCMRepository": func() string { + return getOCMRepository(compGetter) + }, + // getComponentVersionByReference returns a ComponentVersion based on the provided reference name. + // It can take either one or two arguments: + // - One argument: the reference name (string). The search starts from the root component version. + // - Two arguments: the first argument is a ComponentVersion to start the search from, and the second argument is the reference name (string). + // If the ComponentVersion is not found, it returns nil. + // If the ComponentGetter is nil, it panics. + // If the number of arguments is less than 1, it panics. + "getComponentVersionByReference": func(args ...interface{}) *ocmcli.ComponentVersion { + return getComponentVersionByReference(ctx, compGetter, args...) + }, + // componentVersionAsMap converts a ComponentVersion to a map[string]interface{}. + // If the ComponentVersion is nil, it returns nil. + "componentVersionAsMap": func(cv *ocmcli.ComponentVersion) map[string]interface{} { + return componentVersionAsMap(cv) + }, + // getResourceFromComponentVersion retrieves a resource from the given ComponentVersion by its name. + // It takes two arguments: + // - cv: the ComponentVersion from which to retrieve the resource. + // - resourceName: the name of the resource to retrieve (string). + // It returns the resource as a map[string]interface{} or nil if not found. + // If the ComponentGetter is nil, it panics. + // If the resource is not found or an error occurs, it returns nil. + "getResourceFromComponentVersion": func(cv *ocmcli.ComponentVersion, resourceName string) map[string]interface{} { + return getResourceFromComponentVersion(compGetter, cv, resourceName) + }, + // listComponentVersions lists all available versions of the given ComponentVersion's component. + // It takes one argument: + // - cv: the ComponentVersion for which to list available versions. + // It returns a slice of version strings or nil if an error occurs. + // If the ComponentGetter is nil, it panics. + // If an error occurs while listing versions, it returns nil. + "listComponentVersions": func(cv *ocmcli.ComponentVersion) []string { + return listComponentVersions(ctx, compGetter, cv) + }, + }) + } + return t +} + // Execute executes the given template with the provided input data. // It returns the rendered template as a byte slice or an error if the execution fails. // The template name is used for error reporting and debugging purposes.