From 04c7e27c3f1a1252fbc7edd45d3de7a7f8e9f709 Mon Sep 17 00:00:00 2001 From: Dennis Leon Date: Thu, 11 Aug 2022 12:05:07 -0700 Subject: [PATCH] feat: Surface App resources associated to a deploy (#799) - Uses kapp metadata file to list on the AppCR status the app label, namespaces and GK's Signed-off-by: Neil Hickey Signed-off-by: Neil Hickey --- config/crds.yml | 54 +++++++ pkg/apis/kappctrl/v1alpha1/types.go | 16 ++ .../v1alpha1/zz_generated.deepcopy.go | 48 ++++++ pkg/app/app.go | 1 + pkg/app/app_deploy.go | 11 ++ pkg/app/app_reconcile.go | 21 ++- pkg/deploy/kapp.go | 60 +++++++- test/e2e/kappcontroller/app_status_test.go | 144 ++++++++++++++++++ test/e2e/kappcontroller/cancel_test.go | 1 + test/e2e/kappcontroller/git_test.go | 3 + test/e2e/kappcontroller/helm_test.go | 1 + test/e2e/kappcontroller/http_test.go | 1 + test/e2e/kappcontroller/imgpkg_bundle_test.go | 1 + .../e2e/kappcontroller/packageinstall_test.go | 1 + 14 files changed, 360 insertions(+), 3 deletions(-) diff --git a/config/crds.yml b/config/crds.yml index 0a74df835..69fe51b1f 100644 --- a/config/crds.yml +++ b/config/crds.yml @@ -1185,6 +1185,33 @@ spec: type: integer finished: type: boolean + kapp: + description: KappDeployStatus contains the associated AppCR deployed resources + properties: + associatedResources: + description: AssociatedResources contains the associated App label, namespaces and GKs + properties: + groupKinds: + items: + description: GroupKind specifies a Group and a Kind, but does not force a version. This is useful for identifying concepts during lookup stages without having partially valid types + properties: + group: + type: string + kind: + type: string + required: + - group + - kind + type: object + type: array + label: + type: string + namespaces: + items: + type: string + type: array + type: object + type: object startedAt: format: date-time type: string @@ -1656,6 +1683,33 @@ spec: type: integer finished: type: boolean + kapp: + description: KappDeployStatus contains the associated AppCR deployed resources + properties: + associatedResources: + description: AssociatedResources contains the associated App label, namespaces and GKs + properties: + groupKinds: + items: + description: GroupKind specifies a Group and a Kind, but does not force a version. This is useful for identifying concepts during lookup stages without having partially valid types + properties: + group: + type: string + kind: + type: string + required: + - group + - kind + type: object + type: array + label: + type: string + namespaces: + items: + type: string + type: array + type: object + type: object startedAt: format: date-time type: string diff --git a/pkg/apis/kappctrl/v1alpha1/types.go b/pkg/apis/kappctrl/v1alpha1/types.go index 336bb0cda..ead900af1 100644 --- a/pkg/apis/kappctrl/v1alpha1/types.go +++ b/pkg/apis/kappctrl/v1alpha1/types.go @@ -162,6 +162,22 @@ type AppStatusDeploy struct { StartedAt metav1.Time `json:"startedAt,omitempty"` // +optional UpdatedAt metav1.Time `json:"updatedAt,omitempty"` + // +optional + KappDeployStatus *KappDeployStatus `json:"kapp,omitempty"` +} + +// KappDeployStatus contains the associated AppCR deployed resources +// +protobuf=false +type KappDeployStatus struct { + AssociatedResources AssociatedResources `json:"associatedResources,omitempty"` +} + +// AssociatedResources contains the associated App label, namespaces and GKs +// +protobuf=false +type AssociatedResources struct { + Label string `json:"label,omitempty"` + Namespaces []string `json:"namespaces,omitempty"` + GroupKinds []metav1.GroupKind `json:"groupKinds,omitempty"` } // +protobuf=false diff --git a/pkg/apis/kappctrl/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kappctrl/v1alpha1/zz_generated.deepcopy.go index 2b8d6286d..c8d42c15d 100644 --- a/pkg/apis/kappctrl/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kappctrl/v1alpha1/zz_generated.deepcopy.go @@ -572,6 +572,11 @@ func (in *AppStatusDeploy) DeepCopyInto(out *AppStatusDeploy) { *out = *in in.StartedAt.DeepCopyInto(&out.StartedAt) in.UpdatedAt.DeepCopyInto(&out.UpdatedAt) + if in.KappDeployStatus != nil { + in, out := &in.KappDeployStatus, &out.KappDeployStatus + *out = new(KappDeployStatus) + (*in).DeepCopyInto(*out) + } return } @@ -1003,6 +1008,32 @@ func (in *AppTemplateYtt) DeepCopy() *AppTemplateYtt { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AssociatedResources) DeepCopyInto(out *AssociatedResources) { + *out = *in + if in.Namespaces != nil { + in, out := &in.Namespaces, &out.Namespaces + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.GroupKinds != nil { + in, out := &in.GroupKinds, &out.GroupKinds + *out = make([]v1.GroupKind, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AssociatedResources. +func (in *AssociatedResources) DeepCopy() *AssociatedResources { + if in == nil { + return nil + } + out := new(AssociatedResources) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Condition) DeepCopyInto(out *Condition) { *out = *in @@ -1039,3 +1070,20 @@ func (in *GenericStatus) DeepCopy() *GenericStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KappDeployStatus) DeepCopyInto(out *KappDeployStatus) { + *out = *in + in.AssociatedResources.DeepCopyInto(&out.AssociatedResources) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KappDeployStatus. +func (in *KappDeployStatus) DeepCopy() *KappDeployStatus { + if in == nil { + return nil + } + out := new(KappDeployStatus) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/app/app.go b/pkg/app/app.go index 5b5a19e7b..f8a6d88fe 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -37,6 +37,7 @@ type App struct { pendingStatusUpdate bool flushAllStatusUpdates bool + metadata *deploy.Meta } func NewApp(app v1alpha1.App, hooks Hooks, diff --git a/pkg/app/app_deploy.go b/pkg/app/app_deploy.go index 7e038e0da..38ae329f9 100644 --- a/pkg/app/app_deploy.go +++ b/pkg/app/app_deploy.go @@ -35,6 +35,7 @@ func (a *App) deploy(tplOutput string, changedFunc func(exec.CmdRunResult)) exec } result = kapp.Deploy(tplOutput, a.startFlushingAllStatusUpdates, changedFunc) + a.trySaveMetadata(kapp) default: result.AttachErrorf("%s", fmt.Errorf("Unsupported way to deploy")) @@ -122,6 +123,16 @@ func (a *App) inspect() exec.CmdRunResult { return result } +// trySaveMetadata if unable to save the kapp metadata into an App meta continue and do not fail the deploy. +func (a *App) trySaveMetadata(kapp *ctldep.Kapp) { + meta, err := kapp.InternalAppMeta() + if err != nil { + return + } + + a.metadata = meta +} + func (a *App) newKapp(kapp v1alpha1.AppDeployKapp, cancelCh chan struct{}) (*ctldep.Kapp, error) { genericOpts := ctldep.GenericOpts{Name: a.app.Name, Namespace: a.app.Namespace} diff --git a/pkg/app/app_reconcile.go b/pkg/app/app_reconcile.go index a3894d324..0bb2f65f9 100644 --- a/pkg/app/app_reconcile.go +++ b/pkg/app/app_reconcile.go @@ -164,7 +164,26 @@ func (a *App) updateLastDeploy(result exec.CmdRunResult) exec.CmdRunResult { UpdatedAt: metav1.NewTime(time.Now().UTC()), } - a.updateStatus("marking last deploy") + defer a.updateStatus("marking last deploy") + + if a.metadata == nil { + return result + } + + usedGKs := []metav1.GroupKind{} + for _, gk := range a.metadata.UsedGKs { + usedGKs = append(usedGKs, metav1.GroupKind{ + gk.Group, gk.Kind, + }) + } + + a.app.Status.Deploy.KappDeployStatus = &v1alpha1.KappDeployStatus{ + AssociatedResources: v1alpha1.AssociatedResources{ + Label: fmt.Sprintf("%s=%s", a.metadata.LabelKey, a.metadata.LabelValue), + Namespaces: a.metadata.LastChange.Namespaces, + GroupKinds: usedGKs, + }, + } return result } diff --git a/pkg/deploy/kapp.go b/pkg/deploy/kapp.go index 9d427f3fb..c13edd74f 100644 --- a/pkg/deploy/kapp.go +++ b/pkg/deploy/kapp.go @@ -5,14 +5,18 @@ package deploy import ( "bytes" + "errors" "fmt" + "gopkg.in/yaml.v2" "os" goexec "os/exec" + "path/filepath" "strings" "time" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/exec" + "github.com/vmware-tanzu/carvel-kapp-controller/pkg/memdir" ) const ( @@ -28,6 +32,7 @@ type Kapp struct { globalDeployRawOpts []string cancelCh chan struct{} cmdRunner exec.CmdRunner + appMeta *Meta } var _ Deploy = &Kapp{} @@ -38,7 +43,7 @@ var _ Deploy = &Kapp{} func NewKapp(appSuffix string, opts v1alpha1.AppDeployKapp, genericOpts ProcessedGenericOpts, globalDeployRawOpts []string, cancelCh chan struct{}, cmdRunner exec.CmdRunner) *Kapp { - return &Kapp{appSuffix, opts, genericOpts, globalDeployRawOpts, cancelCh, cmdRunner} + return &Kapp{appSuffix, opts, genericOpts, globalDeployRawOpts, cancelCh, cmdRunner, nil} } // Deploy takes the output from templating, and the app name, @@ -46,7 +51,17 @@ func NewKapp(appSuffix string, opts v1alpha1.AppDeployKapp, genericOpts Processe func (a *Kapp) Deploy(tplOutput string, startedApplyingFunc func(), changedFunc func(exec.CmdRunResult)) exec.CmdRunResult { - args, err := a.addDeployArgs([]string{"deploy", "--prev-app", a.oldManagedName(), "-f", "-"}) + tmpMetadataDir := memdir.NewTmpDir("app_metadata") + defer tmpMetadataDir.Remove() + + err := tmpMetadataDir.Create() + if err != nil { + return exec.NewCmdRunResultWithErr(err) + } + + metadataFile := filepath.Join(tmpMetadataDir.Path(), "app-metadata.yml") + + args, err := a.addDeployArgs([]string{"deploy", "--app-metadata-file-output", metadataFile, "--prev-app", a.oldManagedName(), "-f", "-"}) if err != nil { return exec.NewCmdRunResultWithErr(err) } @@ -65,6 +80,9 @@ func (a *Kapp) Deploy(tplOutput string, startedApplyingFunc func(), result := resultBuf.Copy() result.AttachErrorf("Deploying: %s", err) + a.appMeta = nil + a.trySaveAppMeta(metadataFile) + return result } @@ -125,6 +143,28 @@ func (a *Kapp) Inspect() exec.CmdRunResult { return result } +// Meta contains app meta allowing for an AppCR to surface the associated namespaces and GKs +type Meta struct { + LabelKey string `yaml:"labelKey"` + LabelValue string `yaml:"labelValue"` + LastChange struct { + Namespaces []string `yaml:"namespaces"` + } `yaml:"lastChange"` + UsedGKs []struct { + Group string `yaml:"Group"` + Kind string `yaml:"Kind"` + } `yaml:"usedGKs"` +} + +// InternalAppMeta exposes the internal configmap kapp maintains on every app deploy +func (a *Kapp) InternalAppMeta() (*Meta, error) { + if a.appMeta == nil { + return nil, errors.New("Unable to retrieve kapp internal config map") + } + + return a.appMeta, nil +} + func (a *Kapp) trackCmdOutput(cmd *goexec.Cmd, startedApplyingFunc func(), changedFunc func(exec.CmdRunResult)) (*CmdRunResultBuffer, chan struct{}) { @@ -231,3 +271,19 @@ func (a *Kapp) addGenericArgs(args []string, appName string) ([]string, []string return args, env } + +// trySaveAppMeta if unable to save the kapp configmap metadata, then continue and do not fail the deploy. +func (a *Kapp) trySaveAppMeta(metadataFileName string) { + metadataFile, err := os.ReadFile(metadataFileName) + if err != nil { + return + } + + appMetadata := Meta{} + err = yaml.Unmarshal(metadataFile, &appMetadata) + if err != nil { + return + } + + a.appMeta = &appMetadata +} diff --git a/test/e2e/kappcontroller/app_status_test.go b/test/e2e/kappcontroller/app_status_test.go index f6f283419..672276caa 100644 --- a/test/e2e/kappcontroller/app_status_test.go +++ b/test/e2e/kappcontroller/app_status_test.go @@ -111,6 +111,7 @@ spec: cr.Status.Deploy.UpdatedAt = metav1.Time{} cr.Status.Deploy.Stdout = "" cr.Status.Deploy.Stderr = "" + cr.Status.Deploy.KappDeployStatus = nil // fetch cr.Status.Fetch.StartedAt = metav1.Time{} @@ -124,7 +125,150 @@ spec: // template cr.Status.Template.UpdatedAt = metav1.Time{} cr.Status.Template.Stderr = "" + } require.Equal(t, expectedStatus, cr.Status) } + +func Test_AppStatus_Displays_Deploy_Status(t *testing.T) { + env := e2e.BuildEnv(t) + logger := e2e.Logger{} + kapp := e2e.Kapp{t, env.Namespace, logger} + sas := e2e.ServiceAccounts{env.Namespace} + + name := "app-deploy" + appYaml := fmt.Sprintf(` +--- +apiVersion: kappctrl.k14s.io/v1alpha1 +kind: App +metadata: + name: %s + annotations: + kapp.k14s.io/change-group: kappctrl-e2e.k14s.io/apps +spec: + serviceAccountName: kappctrl-e2e-ns-sa + fetch: + - inline: + paths: + file.yml: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: test-app-status-role + template: + - ytt: {} + deploy: + - kapp: + intoNs: %s`, name, env.Namespace) + sas.ForNamespaceYAML() + + cleanUpApp := func() { + kapp.Run([]string{"delete", "-a", name}) + } + + cleanUpApp() + defer cleanUpApp() + + logger.Section("deploy", func() { + kapp.RunWithOpts([]string{"deploy", "-f", "-", "-a", name}, e2e.RunOpts{StdinReader: strings.NewReader(appYaml)}) + }) + + out := kapp.Run([]string{"inspect", "-a", name, "--raw", "--tty=false", "--filter-kind=App"}) + assert.Greater(t, len(out), 1000) // the output yaml should be non-trivial (observed len ~2.7k) + + var cr v1alpha1.App + err := yaml.Unmarshal([]byte(out), &cr) + if err != nil { + t.Fatalf("Failed to unmarshal: %s", err) + } + + expectedAppLabel := kapp.Run([]string{"label", "-a", name + ".app", "--tty=false"}) + + expectedStatus := v1alpha1.AppStatus{ + Deploy: &v1alpha1.AppStatusDeploy{ + KappDeployStatus: &v1alpha1.KappDeployStatus{ + AssociatedResources: v1alpha1.AssociatedResources{ + Label: expectedAppLabel, + Namespaces: []string{"kappctrl-test"}, + GroupKinds: []metav1.GroupKind{ + {Group: "rbac.authorization.k8s.io", Kind: "Role"}, + }, + }, + }, + }, + } + + require.Equal(t, expectedStatus.Deploy.KappDeployStatus, cr.Status.Deploy.KappDeployStatus) + + appYaml = fmt.Sprintf(` +--- +apiVersion: kappctrl.k14s.io/v1alpha1 +kind: App +metadata: + name: %s + annotations: + kapp.k14s.io/change-group: kappctrl-e2e.k14s.io/apps +spec: + serviceAccountName: kappctrl-e2e-ns-sa + fetch: + - inline: + paths: + file.yml: | + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: configmap + data: + key: value + --- + apiVersion: batch/v1 + kind: Job + metadata: + name: fail + spec: + template: + spec: + containers: + - name: fail + image: busybox + command: ["/bin/sh", "-c", "exit 1"] + restartPolicy: Never + backoffLimit: 1 + template: + - ytt: {} + deploy: + - kapp: + intoNs: %s`, name, env.Namespace) + sas.ForNamespaceYAML() + + logger.Section("deploy", func() { + kapp.RunWithOpts([]string{"deploy", "-f", "-", "-a", name}, e2e.RunOpts{StdinReader: strings.NewReader(appYaml), AllowError: true}) + }) + + out = kapp.Run([]string{"inspect", "-a", name, "--raw", "--tty=false", "--filter-kind=App"}) + assert.Greater(t, len(out), 1000) // the output yaml should be non-trivial (observed len ~2.7k) + + var updateCr v1alpha1.App + err = yaml.Unmarshal([]byte(out), &updateCr) + if err != nil { + t.Fatalf("Failed to unmarshal: %s", err) + } + + expectedStatus = v1alpha1.AppStatus{ + Deploy: &v1alpha1.AppStatusDeploy{ + KappDeployStatus: &v1alpha1.KappDeployStatus{ + AssociatedResources: v1alpha1.AssociatedResources{ + Label: expectedAppLabel, + Namespaces: []string{"kappctrl-test"}, + GroupKinds: []metav1.GroupKind{ + {Group: "", Kind: "ConfigMap"}, + {Group: "batch", Kind: "Job"}, + {Group: "rbac.authorization.k8s.io", Kind: "Role"}, + }, + }, + }, + }, + } + + require.Equal(t, expectedStatus.Deploy.KappDeployStatus, updateCr.Status.Deploy.KappDeployStatus) +} diff --git a/test/e2e/kappcontroller/cancel_test.go b/test/e2e/kappcontroller/cancel_test.go index d265524d9..4f154554a 100644 --- a/test/e2e/kappcontroller/cancel_test.go +++ b/test/e2e/kappcontroller/cancel_test.go @@ -158,6 +158,7 @@ data: {} app.Status.Deploy.StartedAt = metav1.Time{} app.Status.Deploy.UpdatedAt = metav1.Time{} app.Status.Deploy.Stdout = "" + app.Status.Deploy.KappDeployStatus = nil if !reflect.DeepEqual(expectedDeploy, app.Status.Deploy) { t.Fatalf("Status deploy is not same: %#v vs %#v", expectedDeploy, app.Status.Deploy) diff --git a/test/e2e/kappcontroller/git_test.go b/test/e2e/kappcontroller/git_test.go index fbc1c81da..ce70a58dd 100644 --- a/test/e2e/kappcontroller/git_test.go +++ b/test/e2e/kappcontroller/git_test.go @@ -99,6 +99,7 @@ spec: cr.Status.Deploy.StartedAt = metav1.Time{} cr.Status.Deploy.UpdatedAt = metav1.Time{} cr.Status.Deploy.Stdout = "" + cr.Status.Deploy.KappDeployStatus = nil // fetch if !strings.Contains(cr.Status.Fetch.Stdout, "kind: LockConfig") { @@ -112,6 +113,7 @@ spec: if !strings.Contains(cr.Status.Inspect.Stdout, "Resources in app 'test-git-https-public.app'") { t.Fatalf("Expected non-empty inspect output") } + cr.Status.Inspect.UpdatedAt = metav1.Time{} cr.Status.Inspect.Stdout = "" @@ -216,6 +218,7 @@ spec: cr.Status.Deploy.StartedAt = metav1.Time{} cr.Status.Deploy.UpdatedAt = metav1.Time{} cr.Status.Deploy.Stdout = "" + cr.Status.Deploy.KappDeployStatus = nil // fetch if !strings.Contains(cr.Status.Fetch.Stdout, "kind: LockConfig") { diff --git a/test/e2e/kappcontroller/helm_test.go b/test/e2e/kappcontroller/helm_test.go index 3ddc2a4bf..0ea24a2ab 100644 --- a/test/e2e/kappcontroller/helm_test.go +++ b/test/e2e/kappcontroller/helm_test.go @@ -125,6 +125,7 @@ stringData: cr.Status.Deploy.StartedAt = metav1.Time{} cr.Status.Deploy.UpdatedAt = metav1.Time{} cr.Status.Deploy.Stdout = "" + cr.Status.Deploy.KappDeployStatus = nil // fetch if !strings.Contains(cr.Status.Fetch.Stdout, "kind: LockConfig") { diff --git a/test/e2e/kappcontroller/http_test.go b/test/e2e/kappcontroller/http_test.go index a485e981c..0e82e1ef1 100644 --- a/test/e2e/kappcontroller/http_test.go +++ b/test/e2e/kappcontroller/http_test.go @@ -97,6 +97,7 @@ spec: cr.Status.Deploy.StartedAt = metav1.Time{} cr.Status.Deploy.UpdatedAt = metav1.Time{} cr.Status.Deploy.Stdout = "" + cr.Status.Deploy.KappDeployStatus = nil // fetch if !strings.Contains(cr.Status.Fetch.Stdout, "kind: LockConfig") { diff --git a/test/e2e/kappcontroller/imgpkg_bundle_test.go b/test/e2e/kappcontroller/imgpkg_bundle_test.go index 05fe8453e..4b32ebc9e 100644 --- a/test/e2e/kappcontroller/imgpkg_bundle_test.go +++ b/test/e2e/kappcontroller/imgpkg_bundle_test.go @@ -98,6 +98,7 @@ spec: cr.Status.Deploy.StartedAt = metav1.Time{} cr.Status.Deploy.UpdatedAt = metav1.Time{} cr.Status.Deploy.Stdout = "" + cr.Status.Deploy.KappDeployStatus = nil // fetch if !strings.Contains(cr.Status.Fetch.Stdout, "- imgpkgBundle") { diff --git a/test/e2e/kappcontroller/packageinstall_test.go b/test/e2e/kappcontroller/packageinstall_test.go index 4b0a3f14e..8973420ff 100644 --- a/test/e2e/kappcontroller/packageinstall_test.go +++ b/test/e2e/kappcontroller/packageinstall_test.go @@ -153,6 +153,7 @@ stringData: cr.Status.Deploy.StartedAt = metav1.Time{} cr.Status.Deploy.UpdatedAt = metav1.Time{} cr.Status.Deploy.Stdout = "" + cr.Status.Deploy.KappDeployStatus = nil // fetch cr.Status.Fetch.StartedAt = metav1.Time{}