From c8fc3d2091242a7f91c3cc0657a1bd92cc88c771 Mon Sep 17 00:00:00 2001 From: Jingfang Liu Date: Mon, 16 Nov 2020 08:40:41 -0800 Subject: [PATCH] split the provider interface --- commands/livecmd.go | 15 +- commands/migratecmd.go | 33 ++-- commands/migratecmd_test.go | 17 ++- go.mod | 2 + go.sum | 2 - internal/cmdsearch/putpattern_test.go | 14 ++ internal/cmdsearch/searchreplace_test.go | 14 ++ pkg/live/dual-delegating-loader.go | 176 ++++++++++++++++++++++ pkg/live/dual-delegating-loader_test.go | 147 ++++++++++++++++++ pkg/live/dual-delegating-provider.go | 122 +++------------ pkg/live/dual-delegating-provider_test.go | 151 ------------------- pkg/live/inventoryrg.go | 32 ++++ pkg/live/rgloader.go | 85 +++++++++++ pkg/live/rgloader_test.go | 15 ++ pkg/live/rgprovider.go | 2 +- 15 files changed, 549 insertions(+), 278 deletions(-) create mode 100644 pkg/live/dual-delegating-loader.go create mode 100644 pkg/live/dual-delegating-loader_test.go create mode 100644 pkg/live/rgloader.go create mode 100644 pkg/live/rgloader_test.go diff --git a/commands/livecmd.go b/commands/livecmd.go index f64d9541e3..64703769b9 100644 --- a/commands/livecmd.go +++ b/commands/livecmd.go @@ -30,6 +30,7 @@ import ( "sigs.k8s.io/cli-utils/cmd/initcmd" "sigs.k8s.io/cli-utils/cmd/preview" "sigs.k8s.io/cli-utils/cmd/status" + "sigs.k8s.io/cli-utils/pkg/manifestreader" "sigs.k8s.io/cli-utils/pkg/provider" ) @@ -65,9 +66,11 @@ func GetLiveCommand(name string, f util.Factory) *cobra.Command { // inventory objects is used. If a package has both inventory objects, then // an error is thrown. var p provider.Provider = provider.NewProvider(f) + var l manifestreader.ManifestLoader = manifestreader.NewManifestLoader(f) if _, exists := os.LookupEnv(resourceGroupEnv); exists { klog.V(2).Infoln("provider supports ResourceGroup and ConfigMap inventory") p = live.NewDualDelegatingProvider(f) + l = live.NewDualDelegatingManifestReader(f) } // The default init command creates the ConfigMap inventory yaml. If the magic @@ -82,13 +85,13 @@ func GetLiveCommand(name string, f util.Factory) *cobra.Command { initCmd.Long = livedocs.InitShort + "\n" + livedocs.InitLong initCmd.Example = livedocs.InitExamples - applyCmd := apply.GetApplyRunner(p, ioStreams).Command + applyCmd := apply.GetApplyRunner(p, l, ioStreams).Command _ = applyCmd.Flags().MarkHidden("no-prune") applyCmd.Short = livedocs.ApplyShort applyCmd.Long = livedocs.ApplyShort + "\n" + livedocs.ApplyLong applyCmd.Example = livedocs.ApplyExamples - previewCmd := preview.GetPreviewRunner(p, ioStreams).Command + previewCmd := preview.GetPreviewRunner(p, l, ioStreams).Command previewCmd.Short = livedocs.PreviewShort previewCmd.Long = livedocs.PreviewShort + "\n" + livedocs.PreviewLong previewCmd.Example = livedocs.PreviewExamples @@ -98,12 +101,12 @@ func GetLiveCommand(name string, f util.Factory) *cobra.Command { diffCmd.Long = livedocs.DiffShort + "\n" + livedocs.DiffLong diffCmd.Example = livedocs.DiffExamples - destroyCmd := destroy.GetDestroyRunner(p, ioStreams).Command + destroyCmd := destroy.GetDestroyRunner(p, l, ioStreams).Command destroyCmd.Short = livedocs.DestroyShort destroyCmd.Long = livedocs.DestroyShort + "\n" + livedocs.DestroyLong destroyCmd.Example = livedocs.DestroyExamples - statusCmd := status.GetStatusRunner(p).Command + statusCmd := status.GetStatusRunner(p, l).Command statusCmd.Short = livedocs.StatusShort statusCmd.Long = livedocs.StatusLong statusCmd.Example = livedocs.StatusExamples @@ -121,7 +124,9 @@ func GetLiveCommand(name string, f util.Factory) *cobra.Command { // migrate command, and add the migrate command to live command. cmProvider := provider.NewProvider(f) rgProvider := live.NewResourceGroupProvider(f) - migrateCmd := GetMigrateRunner(cmProvider, rgProvider, ioStreams).Command + cmLoader := manifestreader.NewManifestLoader(f) + rgLoader := live.NewResourceGroupManifestLoader(f) + migrateCmd := GetMigrateRunner(cmProvider, rgProvider, cmLoader, rgLoader, ioStreams).Command liveCmd.AddCommand(migrateCmd) } diff --git a/commands/migratecmd.go b/commands/migratecmd.go index 8ed8fc0112..f6fb8a4dcc 100644 --- a/commands/migratecmd.go +++ b/commands/migratecmd.go @@ -24,6 +24,7 @@ import ( "sigs.k8s.io/cli-utils/pkg/common" "sigs.k8s.io/cli-utils/pkg/config" "sigs.k8s.io/cli-utils/pkg/inventory" + "sigs.k8s.io/cli-utils/pkg/manifestreader" "sigs.k8s.io/cli-utils/pkg/object" "sigs.k8s.io/cli-utils/pkg/provider" "sigs.k8s.io/kustomize/kyaml/yaml" @@ -39,16 +40,22 @@ type MigrateRunner struct { initOptions *KptInitOptions cmProvider provider.Provider rgProvider provider.Provider + cmLoader manifestreader.ManifestLoader + rgLoader manifestreader.ManifestLoader } // NewMigrateRunner returns a pointer to an initial MigrateRunner structure. -func GetMigrateRunner(cmProvider provider.Provider, rgProvider provider.Provider, ioStreams genericclioptions.IOStreams) *MigrateRunner { +func GetMigrateRunner(cmProvider provider.Provider, rgProvider provider.Provider, + cmLoader manifestreader.ManifestLoader, rgLoader manifestreader.ManifestLoader, + ioStreams genericclioptions.IOStreams) *MigrateRunner { r := &MigrateRunner{ ioStreams: ioStreams, dryRun: false, initOptions: NewKptInitOptions(cmProvider.Factory(), ioStreams), cmProvider: cmProvider, rgProvider: rgProvider, + cmLoader: cmLoader, + rgLoader: rgLoader, dir: "", } cmd := &cobra.Command{ @@ -78,7 +85,9 @@ func GetMigrateRunner(cmProvider provider.Provider, rgProvider provider.Provider func NewCmdMigrate(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) *cobra.Command { configMapProvider := provider.NewProvider(f) resourceGroupProvider := live.NewResourceGroupProvider(f) - return GetMigrateRunner(configMapProvider, resourceGroupProvider, ioStreams).Command + cmLoader := manifestreader.NewManifestLoader(f) + rgLoader := live.NewResourceGroupManifestLoader(f) + return GetMigrateRunner(configMapProvider, resourceGroupProvider, cmLoader, rgLoader, ioStreams).Command } // Run executes the migration from the ConfigMap based inventory to the ResourceGroup @@ -219,9 +228,9 @@ func (mr *MigrateRunner) updateKptfile(args []string) error { // retrieveConfigMapInv retrieves the ConfigMap inventory object or // an error if one occurred. -func (mr *MigrateRunner) retrieveConfigMapInv(reader io.Reader, args []string) (*unstructured.Unstructured, error) { +func (mr *MigrateRunner) retrieveConfigMapInv(reader io.Reader, args []string) (inventory.InventoryInfo, error) { fmt.Fprint(mr.ioStreams.Out, " retrieve the current ConfigMap inventory...") - cmReader, err := mr.cmProvider.ManifestReader(reader, args) + cmReader, err := mr.cmLoader.ManifestReader(reader, args) if err != nil { return nil, err } @@ -229,17 +238,14 @@ func (mr *MigrateRunner) retrieveConfigMapInv(reader io.Reader, args []string) ( if err != nil { return nil, err } - cmInv, _, err := inventory.SplitUnstructureds(objs) - if err != nil { - return nil, err - } - return cmInv, nil + cmInv, _, err := mr.cmLoader.InventoryInfo(objs) + return cmInv, err } // retrieveInvObjs returns the object references from the passed // inventory object by querying the inventory object in the cluster, // or an error if one occurred. -func (mr *MigrateRunner) retrieveInvObjs(invObj *unstructured.Unstructured) ([]object.ObjMetadata, error) { +func (mr *MigrateRunner) retrieveInvObjs(invObj inventory.InventoryInfo) ([]object.ObjMetadata, error) { cmInvClient, err := mr.cmProvider.InventoryClient() if err != nil { return nil, err @@ -261,7 +267,7 @@ func (mr *MigrateRunner) migrateObjs(cmObjs []object.ObjMetadata, reader io.Read fmt.Fprint(mr.ioStreams.Out, "no inventory objects found\n") return nil } - rgReader, err := mr.rgProvider.ManifestReader(reader, args) + rgReader, err := mr.rgLoader.ManifestReader(reader, args) if err != nil { return err } @@ -278,7 +284,8 @@ func (mr *MigrateRunner) migrateObjs(cmObjs []object.ObjMetadata, reader io.Read if err != nil { return err } - _, err = rgInvClient.Merge(rgInv, cmObjs) + inv := live.WrapInventoryInfoObj(rgInv) + _, err = rgInvClient.Merge(inv, cmObjs) if err != nil { return err } @@ -288,7 +295,7 @@ func (mr *MigrateRunner) migrateObjs(cmObjs []object.ObjMetadata, reader io.Read // deleteConfigMapInv removes the passed inventory object from the // cluster. Returns an error if one occurred. -func (mr *MigrateRunner) deleteConfigMapInv(invObj *unstructured.Unstructured) error { +func (mr *MigrateRunner) deleteConfigMapInv(invObj inventory.InventoryInfo) error { fmt.Fprint(mr.ioStreams.Out, " deleting old ConfigMap inventory object...") cmInvClient, err := mr.cmProvider.InventoryClient() if err != nil { diff --git a/commands/migratecmd_test.go b/commands/migratecmd_test.go index dce521c046..db194b81ad 100644 --- a/commands/migratecmd_test.go +++ b/commands/migratecmd_test.go @@ -16,6 +16,7 @@ import ( "k8s.io/cli-runtime/pkg/genericclioptions" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" "sigs.k8s.io/cli-utils/pkg/common" + "sigs.k8s.io/cli-utils/pkg/manifestreader" "sigs.k8s.io/cli-utils/pkg/object" "sigs.k8s.io/cli-utils/pkg/provider" ) @@ -142,7 +143,9 @@ func TestKptMigrate_updateKptfile(t *testing.T) { // Create MigrateRunner and call "updateKptfile" cmProvider := provider.NewFakeProvider(tf, []object.ObjMetadata{}) rgProvider := live.NewResourceGroupProvider(tf) - migrateRunner := GetMigrateRunner(cmProvider, rgProvider, ioStreams) + cmLoader := manifestreader.NewManifestLoader(tf) + rgLoader := live.NewResourceGroupManifestLoader(tf) + migrateRunner := GetMigrateRunner(cmProvider, rgProvider, cmLoader, rgLoader, ioStreams) migrateRunner.dryRun = tc.dryRun err = migrateRunner.updateKptfile([]string{dir}) // Check if there should be an error @@ -207,7 +210,9 @@ func TestKptMigrate_retrieveConfigMapInv(t *testing.T) { // Create MigrateRunner and call "retrieveConfigMapInv" cmProvider := provider.NewFakeProvider(tf, []object.ObjMetadata{}) rgProvider := live.NewResourceGroupProvider(tf) - migrateRunner := GetMigrateRunner(cmProvider, rgProvider, ioStreams) + cmLoader := manifestreader.NewManifestLoader(tf) + rgLoader := live.NewResourceGroupManifestLoader(tf) + migrateRunner := GetMigrateRunner(cmProvider, rgProvider, cmLoader, rgLoader, ioStreams) actual, err := migrateRunner.retrieveConfigMapInv(strings.NewReader(tc.configMap), []string{}) // Check if there should be an error if tc.isError { @@ -217,10 +222,10 @@ func TestKptMigrate_retrieveConfigMapInv(t *testing.T) { return } assert.NoError(t, err) - if tc.expected.GetName() != actual.GetName() { + if tc.expected.GetName() != actual.Name() { t.Errorf("expected ConfigMap (%#v), got (%#v)", tc.expected, actual) } - if tc.expected.GetNamespace() != actual.GetNamespace() { + if tc.expected.GetNamespace() != actual.Namespace() { t.Errorf("expected ConfigMap (%#v), got (%#v)", tc.expected, actual) } }) @@ -308,7 +313,9 @@ func TestKptMigrate_migrateObjs(t *testing.T) { // Create MigrateRunner and call "retrieveConfigMapInv" cmProvider := provider.NewFakeProvider(tf, []object.ObjMetadata{}) rgProvider := live.NewFakeResourceGroupProvider(tf, tc.objs) - migrateRunner := GetMigrateRunner(cmProvider, rgProvider, ioStreams) + cmLoader := manifestreader.NewManifestLoader(tf) + rgLoader := live.NewResourceGroupManifestLoader(tf) + migrateRunner := GetMigrateRunner(cmProvider, rgProvider, cmLoader, rgLoader, ioStreams) err := migrateRunner.migrateObjs(tc.objs, strings.NewReader(tc.invObj), []string{}) // Check if there should be an error if tc.isError { diff --git a/go.mod b/go.mod index 7d298cbce7..dabff996b2 100644 --- a/go.mod +++ b/go.mod @@ -26,3 +26,5 @@ require ( sigs.k8s.io/kustomize/cmd/config v0.8.5 sigs.k8s.io/kustomize/kyaml v0.9.4 ) + +replace sigs.k8s.io/cli-utils => ../../sigs.k8s.io/cli-utils diff --git a/go.sum b/go.sum index 93298cd9b7..8eb67e735c 100644 --- a/go.sum +++ b/go.sum @@ -683,8 +683,6 @@ k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTU k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= -sigs.k8s.io/cli-utils v0.21.1 h1:cDNLRGvVshr9XjpaP0PghMXUQOLB4QFtZC+/YTGiWyw= -sigs.k8s.io/cli-utils v0.21.1/go.mod h1:Mt1gLc/Nfa7Z3Lhbfk72uT2Kc4GNyuX4oMqEN9FbPMs= sigs.k8s.io/controller-runtime v0.6.0 h1:Fzna3DY7c4BIP6KwfSlrfnj20DJ+SeMBK8HSFvOk9NM= sigs.k8s.io/controller-runtime v0.6.0/go.mod h1:CpYf5pdNY/B352A1TFLAS2JVSlnGQ5O2cftPHndTroo= sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= diff --git a/internal/cmdsearch/putpattern_test.go b/internal/cmdsearch/putpattern_test.go index 33a295679b..793c59816b 100644 --- a/internal/cmdsearch/putpattern_test.go +++ b/internal/cmdsearch/putpattern_test.go @@ -1,3 +1,17 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmdsearch var putPatternCases = []test{ diff --git a/internal/cmdsearch/searchreplace_test.go b/internal/cmdsearch/searchreplace_test.go index 2f1f5cd473..7a8b874f25 100644 --- a/internal/cmdsearch/searchreplace_test.go +++ b/internal/cmdsearch/searchreplace_test.go @@ -1,3 +1,17 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmdsearch var searchReplaceCases = []test{ diff --git a/pkg/live/dual-delegating-loader.go b/pkg/live/dual-delegating-loader.go new file mode 100644 index 0000000000..17f58ca6c8 --- /dev/null +++ b/pkg/live/dual-delegating-loader.go @@ -0,0 +1,176 @@ +// Copyright 2020 Google LLC. +// SPDX-License-Identifier: Apache-2.0 + +package live + +import ( + "fmt" + "io" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/klog" + "k8s.io/kubectl/pkg/cmd/util" + "sigs.k8s.io/cli-utils/pkg/inventory" + "sigs.k8s.io/cli-utils/pkg/manifestreader" +) + +var _ manifestreader.ManifestLoader = &DualDelegatingManifestReader{} + +// DualDelegatingManifestReader read manifests that either uses ConfigMap +// or the ResourceGroup as the inventory object. +type DualDelegatingManifestReader struct { + factory util.Factory +} + +var _ manifestreader.ManifestLoader = &DualDelegatingManifestReader{} + +func NewDualDelegatingManifestReader(f util.Factory) *DualDelegatingManifestReader { + return &DualDelegatingManifestReader{factory: f} +} + +// ManifestReader retrieves the ManifestReader from the delegate ResourceGroup +// Provider, then calls Read() for this ManifestReader to retrieve the objects +// and to calculate the type of Inventory object is present. Returns a +// CachedManifestReader with the read objects, or an error. +func (cp *DualDelegatingManifestReader) ManifestReader(reader io.Reader, args []string) (manifestreader.ManifestReader, error) { + r, err := cp.manifestReader(reader, args) + if err != nil { + return nil, err + } + objs, err := r.Read() + if err != nil { + return nil, err + } + klog.V(4).Infof("ManifestReader read %d objects", len(objs)) + return &CachedManifestReader{objs: objs}, nil +} + +func (cp *DualDelegatingManifestReader) manifestReader(reader io.Reader, args []string) (manifestreader.ManifestReader, error) { + // Validate parameters. + if reader == nil && len(args) == 0 { + return nil, fmt.Errorf("unable to build ManifestReader without both reader or args") + } + if len(args) > 1 { + return nil, fmt.Errorf("expected one directory argument allowed; got (%s)", args) + } + // Create ReaderOptions for subsequent ManifestReader. + namespace, enforceNamespace, err := cp.factory.ToRawKubeConfigLoader().Namespace() + if err != nil { + return nil, err + } + mapper, err := cp.factory.ToRESTMapper() + if err != nil { + return nil, err + } + + readerOptions := manifestreader.ReaderOptions{ + Mapper: mapper, + Namespace: namespace, + EnforceNamespace: enforceNamespace, + } + // No arguments means stream (using reader), while one argument + // means path manifest reader. + var rgReader manifestreader.ManifestReader + if len(args) == 0 { + rgReader = &ResourceGroupStreamManifestReader{ + streamReader: &manifestreader.StreamManifestReader{ + ReaderName: "stdin", + Reader: reader, + ReaderOptions: readerOptions, + }, + } + } else { + rgReader = &ResourceGroupPathManifestReader{ + pathReader: &manifestreader.PathManifestReader{ + Path: args[0], + ReaderOptions: readerOptions, + }, + } + } + return rgReader, nil +} + +// InventoryInfo returns the InventoryInfo from a list of Unstructured objects. +// It can return a NoInventoryError or MultipleInventoryError. +func (cp *DualDelegatingManifestReader) InventoryInfo(objs []*unstructured.Unstructured) (inventory.InventoryInfo, []*unstructured.Unstructured, error) { + objs, rgInv := findResourceGroupInv(objs) + var inv inventory.InventoryInfo + // A ResourceGroup inventory object means we need an InventoryFactoryFunc + // which works for ResourceGroup (instead of ConfigMap, which is default). + if rgInv != nil { + inv = &InventoryResourceGroup{inv: rgInv} + } + objs, cmInv := findConfigMapInv(objs) + if rgInv == nil && cmInv == nil { + return nil, objs, inventory.NoInventoryObjError{} + } + if rgInv != nil && cmInv != nil { + return nil, objs, MultipleInventoryObjError{ + InvObjs: []*unstructured.Unstructured{rgInv, cmInv}, + } + } + if cmInv != nil { + inv = inventory.WrapInventoryInfoObj(cmInv) + } + //cp.calcInventory = true + return inv, objs, nil +} + +// MultipleInventoryObjError is thrown when more than one inventory +// objects is detected. +type MultipleInventoryObjError struct { + InvObjs []*unstructured.Unstructured +} + +const multipleInventoryErrorStr = `Detected ResourceGroup (Kptfile) and deprecated ConfigMap (inventory-template.yaml) +inventory objects. Please run "kpt live migrate" to complete upgrade to +ResourceGroup inventory object. +` + +func (g MultipleInventoryObjError) Error() string { + return multipleInventoryErrorStr +} + +// CachedManifestReader implements ManifestReader, storing objects to return. +type CachedManifestReader struct { + objs []*unstructured.Unstructured +} + +// Read simply returns the stored objects. +func (r *CachedManifestReader) Read() ([]*unstructured.Unstructured, error) { + return r.objs, nil +} + +// findResourceGroupInv returns the pointer to the ResourceGroup inventory object, +// or nil if it does not exist. +func findResourceGroupInv(objs []*unstructured.Unstructured) ([]*unstructured.Unstructured, *unstructured.Unstructured) { + var fileteredObjs []*unstructured.Unstructured + var inventoryObj *unstructured.Unstructured + for _, obj := range objs { + if inventory.IsInventoryObject(obj) { + if obj.GetKind() == "ResourceGroup" { + inventoryObj = obj + continue + } + } + fileteredObjs = append(fileteredObjs, obj) + } + return fileteredObjs, inventoryObj +} + +// findConfigMapInv returns the pointer to the ConfigMap inventory object, +// or nil if it does not exist. +func findConfigMapInv(objs []*unstructured.Unstructured) ([]*unstructured.Unstructured, *unstructured.Unstructured) { + var fileteredObjs []*unstructured.Unstructured + var inventoryObj *unstructured.Unstructured + for _, obj := range objs { + if inventory.IsInventoryObject(obj) { + if obj.GetKind() == "ConfigMap" { + inventoryObj = obj + continue + } + } + fileteredObjs = append(fileteredObjs, obj) + } + return fileteredObjs, inventoryObj +} diff --git a/pkg/live/dual-delegating-loader_test.go b/pkg/live/dual-delegating-loader_test.go new file mode 100644 index 0000000000..7cc90ca02e --- /dev/null +++ b/pkg/live/dual-delegating-loader_test.go @@ -0,0 +1,147 @@ +// Copyright 2020 Google LLC. +// SPDX-License-Identifier: Apache-2.0 + +package live + +import ( + "io/ioutil" + "path/filepath" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + "sigs.k8s.io/cli-utils/pkg/inventory" +) + +var configMapInv = ` +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: test-ns + name: inventory-111111 + labels: + cli-utils.sigs.k8s.io/inventory-id: XXXX-YYYY-ZZZZ +` + +func TestDualDelegatingProvider_Read(t *testing.T) { + testCases := map[string]struct { + manifests map[string]string + numObjs int + invKind inventory.InventoryInfo + isError bool + }{ + "Basic ResourceGroup inventory object created": { + manifests: map[string]string{ + "Kptfile": kptFile, + "pod-a.yaml": podA, + }, + numObjs: 2, + invKind: &InventoryResourceGroup{}, + isError: false, + }, + "Only ResourceGroup inventory object created": { + manifests: map[string]string{ + "Kptfile": kptFile, + }, + numObjs: 1, + invKind: &InventoryResourceGroup{}, + isError: false, + }, + "ResourceGroup inventory object with multiple objects": { + manifests: map[string]string{ + "pod-a.yaml": podA, + "Kptfile": kptFile, + "deployment-a.yaml": deploymentA, + }, + numObjs: 3, + invKind: &InventoryResourceGroup{}, + isError: false, + }, + "Basic ConfigMap inventory object created": { + manifests: map[string]string{ + "inventory-template.yaml": configMapInv, + "deployment-a.yaml": deploymentA, + }, + numObjs: 2, + invKind: &inventory.InventoryConfigMap{}, + isError: false, + }, + "Only ConfigMap inventory object created": { + manifests: map[string]string{ + "inventory-template.yaml": configMapInv, + }, + numObjs: 1, + invKind: &inventory.InventoryConfigMap{}, + isError: false, + }, + "ConfigMap inventory object with multiple objects": { + manifests: map[string]string{ + "deployment-a.yaml": deploymentA, + "inventory-template.yaml": configMapInv, + "pod-a.yaml": podA, + }, + numObjs: 3, + invKind: &inventory.InventoryConfigMap{}, + isError: false, + }, + "No inventory manifests is an error": { + manifests: map[string]string{ + "pod-a.yaml": podA, + "deployment-a.yaml": deploymentA, + }, + numObjs: 2, + isError: true, + }, + "Multiple manifests is an error": { + manifests: map[string]string{ + "inventory-template.yaml": configMapInv, + "Kptfile": kptFile, + "pod-a.yaml": podA, + }, + numObjs: 3, + isError: true, + }, + } + + for tn, tc := range testCases { + t.Run(tn, func(t *testing.T) { + // Create the fake factory + tf := cmdtesting.NewTestFactory().WithNamespace("test-ns") + defer tf.Cleanup() + // Set up the yaml manifests (including Kptfile) in temp dir. + dir, err := ioutil.TempDir("", "provider-test") + assert.NoError(t, err) + for filename, content := range tc.manifests { + p := filepath.Join(dir, filename) + err := ioutil.WriteFile(p, []byte(content), 0600) + assert.NoError(t, err) + } + + // Read objects using provider ManifestReader. + loader := NewDualDelegatingManifestReader(tf) + mr, err := loader.ManifestReader(nil, []string{dir}) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + objs, err := mr.Read() + assert.NoError(t, err) + if tc.numObjs != len(objs) { + t.Errorf("expected to read (%d) objs, got (%d)", tc.numObjs, len(objs)) + } + inv, _, err := loader.InventoryInfo(objs) + if tc.isError { + if err == nil { + t.Errorf("expected error on ManifestReader, but received none.") + } + return + } + if inv == nil { + t.Errorf("inventory object not found") + } + if reflect.TypeOf(tc.invKind) != reflect.TypeOf(inv) { + t.Errorf("expected inventory kind (%s), got (%s)", reflect.TypeOf(tc.invKind), reflect.TypeOf(inv)) + } + }) + } +} diff --git a/pkg/live/dual-delegating-provider.go b/pkg/live/dual-delegating-provider.go index 19c9727f86..7ddd84845a 100644 --- a/pkg/live/dual-delegating-provider.go +++ b/pkg/live/dual-delegating-provider.go @@ -4,15 +4,10 @@ package live import ( - "fmt" - "io" - - "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/klog" + "k8s.io/kubectl/pkg/cmd/util" "sigs.k8s.io/cli-utils/pkg/inventory" - "sigs.k8s.io/cli-utils/pkg/manifestreader" "sigs.k8s.io/cli-utils/pkg/provider" ) @@ -25,19 +20,13 @@ var _ provider.Provider = &DualDelegatingProvider{} type DualDelegatingProvider struct { // ResourceGroupProvider is the delegate. rgProvider provider.Provider - // Default inventory function is for ConfigMap. - wrapInv inventory.InventoryFactoryFunc - // Boolean on whether we've calculated the inventory object type. - calcInventory bool } // NewDualDelagatingProvider returns a pointer to the DualDelegatingProvider, // setting default values. func NewDualDelegatingProvider(f util.Factory) *DualDelegatingProvider { return &DualDelegatingProvider{ - rgProvider: NewResourceGroupProvider(f), - wrapInv: inventory.WrapInventoryObj, - calcInventory: false, + rgProvider: NewResourceGroupProvider(f), } } @@ -50,98 +39,29 @@ func (cp *DualDelegatingProvider) Factory() util.Factory { // stored/calculated InventoryFactoryFunction. This must be called // after ManifestReader(). func (cp *DualDelegatingProvider) InventoryClient() (inventory.InventoryClient, error) { - if !cp.calcInventory { - return nil, fmt.Errorf("must be called after ManifestReader()") - } - return inventory.NewInventoryClient(cp.Factory(), cp.wrapInv) -} - -// ToRESTMapper returns the value from the delegate provider; or an error. -func (cp *DualDelegatingProvider) ToRESTMapper() (meta.RESTMapper, error) { - return cp.Factory().ToRESTMapper() -} - -// ManifestReader retrieves the ManifestReader from the delegate ResourceGroup -// Provider, then calls Read() for this ManifestReader to retrieve the objects -// and to calculate the type of Inventory object is present. Returns a -// CachedManifestReader with the read objects, or an error. Can return a -// NoInventoryError or MultipleInventoryError. -func (cp *DualDelegatingProvider) ManifestReader(reader io.Reader, args []string) (manifestreader.ManifestReader, error) { - r, err := cp.rgProvider.ManifestReader(reader, args) - if err != nil { - return nil, err - } - objs, err := r.Read() - if err != nil { - return nil, err - } - klog.V(4).Infof("ManifestReader read %d objects", len(objs)) - rgInv := findResourceGroupInv(objs) - // A ResourceGroup inventory object means we need an InventoryFactoryFunc - // which works for ResourceGroup (instead of ConfigMap, which is default). - if rgInv != nil { - cp.wrapInv = WrapInventoryObj - } - cmInv := findConfigMapInv(objs) - if rgInv == nil && cmInv == nil { - return nil, inventory.NoInventoryObjError{} - } - if rgInv != nil && cmInv != nil { - return nil, MultipleInventoryObjError{ - InvObjs: []*unstructured.Unstructured{rgInv, cmInv}, - } - } - cp.calcInventory = true - return &CachedManifestReader{objs: objs}, nil -} - -// MultipleInventoryObjError is thrown when more than one inventory -// objects is detected. -type MultipleInventoryObjError struct { - InvObjs []*unstructured.Unstructured -} - -const multipleInventoryErrorStr = `Detected ResourceGroup (Kptfile) and deprecated ConfigMap (inventory-template.yaml) -inventory objects. Please run "kpt live migrate" to complete upgrade to -ResourceGroup inventory object. -` - -func (g MultipleInventoryObjError) Error() string { - return multipleInventoryErrorStr -} - -// CachedManifestReader implements ManifestReader, storing objects to return. -type CachedManifestReader struct { - objs []*unstructured.Unstructured -} - -// Read simply returns the stored objects. -func (r *CachedManifestReader) Read() ([]*unstructured.Unstructured, error) { - return r.objs, nil + return inventory.NewInventoryClient(cp.Factory(), + inventroyWrapperFunc, + invToUnstructuredFunc) } -// findResourceGroupInv returns the pointer to the ResourceGroup inventory object, -// or nil if it does not exist. -func findResourceGroupInv(objs []*unstructured.Unstructured) *unstructured.Unstructured { - for _, obj := range objs { - if inventory.IsInventoryObject(obj) { - if obj.GetKind() == "ResourceGroup" { - return obj - } - } +func inventroyWrapperFunc(obj *unstructured.Unstructured) inventory.Inventory { + switch obj.GetKind() { + case "ResourceGroup": + return &InventoryResourceGroup{inv: obj} + case "ConfigMap": + return inventory.WrapInventoryObj(obj) + default: + return nil } - return nil } -// findConfigMapInv returns the pointer to the ConfigMap inventory object, -// or nil if it does not exist. -func findConfigMapInv(objs []*unstructured.Unstructured) *unstructured.Unstructured { - for _, obj := range objs { - if inventory.IsInventoryObject(obj) { - if obj.GetKind() == "ConfigMap" { - return obj - } - } +func invToUnstructuredFunc(inv inventory.InventoryInfo) *unstructured.Unstructured { + switch invInfo := inv.(type) { + case *InventoryResourceGroup: + return invInfo.inv + case *inventory.InventoryConfigMap: + return invInfo.UnstructuredInventory() + default: + return nil } - return nil } diff --git a/pkg/live/dual-delegating-provider_test.go b/pkg/live/dual-delegating-provider_test.go index 220247a365..26cf04c29e 100644 --- a/pkg/live/dual-delegating-provider_test.go +++ b/pkg/live/dual-delegating-provider_test.go @@ -2,154 +2,3 @@ // SPDX-License-Identifier: Apache-2.0 package live - -import ( - "io/ioutil" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - cmdtesting "k8s.io/kubectl/pkg/cmd/testing" - "sigs.k8s.io/cli-utils/pkg/inventory" -) - -var configMapInv = ` -apiVersion: v1 -kind: ConfigMap -metadata: - namespace: test-ns - name: inventory-111111 - labels: - cli-utils.sigs.k8s.io/inventory-id: XXXX-YYYY-ZZZZ -` - -func TestDualDelegatingProvider_Read(t *testing.T) { - testCases := map[string]struct { - manifests map[string]string - numObjs int - invKind string - isError bool - }{ - "Basic ResourceGroup inventory object created": { - manifests: map[string]string{ - "Kptfile": kptFile, - "pod-a.yaml": podA, - }, - numObjs: 2, - invKind: "ResourceGroup", - isError: false, - }, - "Only ResourceGroup inventory object created": { - manifests: map[string]string{ - "Kptfile": kptFile, - }, - numObjs: 1, - invKind: "ResourceGroup", - isError: false, - }, - "ResourceGroup inventory object with multiple objects": { - manifests: map[string]string{ - "pod-a.yaml": podA, - "Kptfile": kptFile, - "deployment-a.yaml": deploymentA, - }, - numObjs: 3, - invKind: "ResourceGroup", - isError: false, - }, - "Basic ConfigMap inventory object created": { - manifests: map[string]string{ - "inventory-template.yaml": configMapInv, - "deployment-a.yaml": deploymentA, - }, - numObjs: 2, - invKind: "ConfigMap", - isError: false, - }, - "Only ConfigMap inventory object created": { - manifests: map[string]string{ - "inventory-template.yaml": configMapInv, - }, - numObjs: 1, - invKind: "ConfigMap", - isError: false, - }, - "ConfigMap inventory object with multiple objects": { - manifests: map[string]string{ - "deployment-a.yaml": deploymentA, - "inventory-template.yaml": configMapInv, - "pod-a.yaml": podA, - }, - numObjs: 3, - invKind: "ConfigMap", - isError: false, - }, - "No inventory manifests is an error": { - manifests: map[string]string{ - "pod-a.yaml": podA, - "deployment-a.yaml": deploymentA, - }, - numObjs: 2, - isError: true, - }, - "Multiple manifests is an error": { - manifests: map[string]string{ - "inventory-template.yaml": configMapInv, - "Kptfile": kptFile, - "pod-a.yaml": podA, - }, - numObjs: 3, - isError: true, - }, - } - - for tn, tc := range testCases { - t.Run(tn, func(t *testing.T) { - // Create the fake factory - tf := cmdtesting.NewTestFactory().WithNamespace("test-ns") - defer tf.Cleanup() - // Set up the yaml manifests (including Kptfile) in temp dir. - dir, err := ioutil.TempDir("", "provider-test") - assert.NoError(t, err) - for filename, content := range tc.manifests { - p := filepath.Join(dir, filename) - err := ioutil.WriteFile(p, []byte(content), 0600) - assert.NoError(t, err) - } - // Create the DualDelegatingProvider - provider := NewDualDelegatingProvider(tf) - assert.Equal(t, tf, provider.Factory()) - // Calling InventoryClient before ManifestReader is always an error. - _, err = provider.InventoryClient() - if err == nil { - t.Errorf("expecting error on InventoryClient, but received none.") - } - // Read objects using provider ManifestReader. - mr, err := provider.ManifestReader(nil, []string{dir}) - if tc.isError { - if err == nil { - t.Errorf("expected error on ManifestReader, but received none.") - } - return - } - objs, err := mr.Read() - assert.NoError(t, err) - if tc.numObjs != len(objs) { - t.Errorf("expected to read (%d) objs, got (%d)", tc.numObjs, len(objs)) - } - // Retrieve single inventory object and validate the kind. - inv := inventory.FindInventoryObj(objs) - if inv == nil { - t.Errorf("inventory object not found") - } - if tc.invKind != inv.GetKind() { - t.Errorf("expected inventory kind (%s), got (%s)", tc.invKind, inv.GetKind()) - } - // Calling InventoryClient after ManifestReader is valid. - _, err = provider.InventoryClient() - if err != nil { - t.Errorf("unexpected error calling InventoryClient: %s", err) - } - }) - } -} diff --git a/pkg/live/inventoryrg.go b/pkg/live/inventoryrg.go index 915baff286..18ac7a9db8 100644 --- a/pkg/live/inventoryrg.go +++ b/pkg/live/inventoryrg.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/klog" + "sigs.k8s.io/cli-utils/pkg/common" "sigs.k8s.io/cli-utils/pkg/inventory" "sigs.k8s.io/cli-utils/pkg/object" ) @@ -41,6 +42,8 @@ type InventoryResourceGroup struct { objMetas []object.ObjMetadata } +var _ inventory.InventoryInfo = &InventoryResourceGroup{} + // WrapInventoryObj takes a passed ResourceGroup (as a resource.Info), // wraps it with the InventoryResourceGroup and upcasts the wrapper as // an the Inventory interface. @@ -49,6 +52,35 @@ func WrapInventoryObj(obj *unstructured.Unstructured) inventory.Inventory { return &InventoryResourceGroup{inv: obj} } +func WrapInventoryInfoObj(obj *unstructured.Unstructured) inventory.InventoryInfo { + return &InventoryResourceGroup{inv: obj} +} + +func InvToUnstructuredFunc(inv inventory.InventoryInfo) *unstructured.Unstructured { + switch invInfo := inv.(type) { + case *InventoryResourceGroup: + return invInfo.inv + default: + return nil + } +} + +func (icm *InventoryResourceGroup) Name() string { + return icm.inv.GetName() +} + +func (icm *InventoryResourceGroup) Namespace() string { + return icm.inv.GetNamespace() +} + +func (icm *InventoryResourceGroup) ID() string { + labels := icm.inv.GetLabels() + if val, found := labels[common.InventoryLabel]; found { + return val + } + return "" +} + // Load is an Inventory interface function returning the set of // object metadata from the wrapped ResourceGroup, or an error. func (icm *InventoryResourceGroup) Load() ([]object.ObjMetadata, error) { diff --git a/pkg/live/rgloader.go b/pkg/live/rgloader.go new file mode 100644 index 0000000000..b15f9beaa1 --- /dev/null +++ b/pkg/live/rgloader.go @@ -0,0 +1,85 @@ +// Copyright 2020 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package live + +import ( + "fmt" + "io" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/kubectl/pkg/cmd/util" + "sigs.k8s.io/cli-utils/pkg/inventory" + "sigs.k8s.io/cli-utils/pkg/manifestreader" +) + +var _ manifestreader.ManifestLoader = &ResourceGroupManifestLoader{} + +// ResourceGroupManifestLoader implements the Provider interface, returning +// ResourceGroup versions of some kpt live apply structures. +type ResourceGroupManifestLoader struct { + factory util.Factory +} + +// NewResourceGroupProvider encapsulates the passed values, and returns a pointer to an ResourceGroupProvider. +func NewResourceGroupManifestLoader(f util.Factory) *ResourceGroupManifestLoader { + return &ResourceGroupManifestLoader{ + factory: f, + } +} + +// Factory returns the kubectl factory. +func (f *ResourceGroupManifestLoader) InventoryInfo(objs []*unstructured.Unstructured) (inventory.InventoryInfo, []*unstructured.Unstructured, error) { + objs, invObj := findResourceGroupInv(objs) + if invObj == nil { + return nil, nil, fmt.Errorf("unable to find the ResourceGroup inventory info") + } + return &InventoryResourceGroup{inv: invObj}, objs, nil +} + +// ManifestReader returns the ResourceGroup inventory object version of +// the ManifestReader. +func (f *ResourceGroupManifestLoader) ManifestReader(reader io.Reader, args []string) (manifestreader.ManifestReader, error) { + // Validate parameters. + if reader == nil && len(args) == 0 { + return nil, fmt.Errorf("unable to build ManifestReader without both reader or args") + } + if len(args) > 1 { + return nil, fmt.Errorf("expected one directory argument allowed; got (%s)", args) + } + // Create ReaderOptions for subsequent ManifestReader. + namespace, enforceNamespace, err := f.factory.ToRawKubeConfigLoader().Namespace() + if err != nil { + return nil, err + } + mapper, err := f.factory.ToRESTMapper() + if err != nil { + return nil, err + } + + readerOptions := manifestreader.ReaderOptions{ + Mapper: mapper, + Namespace: namespace, + EnforceNamespace: enforceNamespace, + } + // No arguments means stream (using reader), while one argument + // means path manifest reader. + var rgReader manifestreader.ManifestReader + if len(args) == 0 { + rgReader = &ResourceGroupStreamManifestReader{ + streamReader: &manifestreader.StreamManifestReader{ + ReaderName: "stdin", + Reader: reader, + ReaderOptions: readerOptions, + }, + } + } else { + rgReader = &ResourceGroupPathManifestReader{ + pathReader: &manifestreader.PathManifestReader{ + Path: args[0], + ReaderOptions: readerOptions, + }, + } + } + return rgReader, nil +} diff --git a/pkg/live/rgloader_test.go b/pkg/live/rgloader_test.go new file mode 100644 index 0000000000..1f9792be31 --- /dev/null +++ b/pkg/live/rgloader_test.go @@ -0,0 +1,15 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package live diff --git a/pkg/live/rgprovider.go b/pkg/live/rgprovider.go index 4751fd1bbd..7ac94ce01a 100644 --- a/pkg/live/rgprovider.go +++ b/pkg/live/rgprovider.go @@ -37,7 +37,7 @@ func (f *ResourceGroupProvider) Factory() util.Factory { // InventoryClient returns the InventoryClient created using the // ResourceGroup inventory object wrapper function. func (f *ResourceGroupProvider) InventoryClient() (inventory.InventoryClient, error) { - return inventory.NewInventoryClient(f.factory, WrapInventoryObj) + return inventory.NewInventoryClient(f.factory, WrapInventoryObj, InvToUnstructuredFunc) } // ToRESTMapper returns the RESTMapper or an erro if one occurred.