Skip to content

Commit

Permalink
Combined Main Commit
Browse files Browse the repository at this point in the history 
Save inventory information into resourcegroup.yaml (kptdev#2615)

Fix site sidebar to also show annotation references (kptdev#2734)

effective customizations chapter (kptdev#2659)

merging this as an MVP and will iterate based on additional requests and customer feedback.

docs: Describe how reconcile status is computed for Config Connector resources (kptdev#2739)
  • Loading branch information
@rquitales @martinmaly
rquitales authored and martinmaly committed Feb 18, 2022
1 parent 37dfaa3 commit cc259eb
Show file tree
Hide file tree
Showing 14 changed files with 700 additions and 13 deletions.
149 changes: 149 additions & 0 deletions internal/cmdliveinit/cmdliveinit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,11 @@ package cmdliveinit
import (
"context"
"crypto/sha1"
goerrors "errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"time"
Expand All @@ -16,16 +19,19 @@ import (
"github.com/GoogleContainerTools/kpt/internal/errors"
"github.com/GoogleContainerTools/kpt/internal/pkg"
"github.com/GoogleContainerTools/kpt/internal/printer"
"github.com/GoogleContainerTools/kpt/internal/types"
"github.com/GoogleContainerTools/kpt/internal/util/attribution"
"github.com/GoogleContainerTools/kpt/internal/util/pathutil"
kptfilev1 "github.com/GoogleContainerTools/kpt/pkg/api/kptfile/v1"
rgfilev1alpha1 "github.com/GoogleContainerTools/kpt/pkg/api/resourcegroup/v1alpha1"
"github.com/GoogleContainerTools/kpt/pkg/kptfile/kptfileutil"
"github.com/spf13/cobra"
"k8s.io/cli-runtime/pkg/genericclioptions"
k8scmdutil "k8s.io/kubectl/pkg/cmd/util"
"sigs.k8s.io/cli-utils/pkg/common"
"sigs.k8s.io/cli-utils/pkg/config"
"sigs.k8s.io/kustomize/kyaml/filesys"
"sigs.k8s.io/kustomize/kyaml/yaml"
)

const defaultInventoryName = "inventory"
Expand All @@ -38,6 +44,24 @@ func (i *InvExistsError) Error() string {
return "inventory information already set for package"
}

// InvInRGExistsError defines new error when the inventory
// values have already been set on the ResourceGroup file and we will warn
// the user to migrate rather than init. This is part of kpt live STDIN work.
type InvInRGExistsError struct{}

func (i *InvInRGExistsError) Error() string {
return "inventory information already set for package"
}

// InvInKfExistsError defines new error when the inventory
// values have already been set on the Kptfile and we will warn
// the user to migrate rather than init. This is part of kpt live STDIN work.
type InvInKfExistsError struct{}

func (i *InvInKfExistsError) Error() string {
return "inventory information already set within Kptfile for package"
}

func NewRunner(ctx context.Context, factory k8scmdutil.Factory,
ioStreams genericclioptions.IOStreams) *Runner {
r := &Runner{
Expand Down Expand Up @@ -76,6 +100,7 @@ type Runner struct {
Force bool // Set inventory values even if already set in Kptfile
Name string // Inventory object name
namespace string // Inventory object namespace
RGFile string // resourcegroup object filepath
InventoryID string // Inventory object unique identifier label
Quiet bool // Output message during initialization
}
Expand Down Expand Up @@ -112,6 +137,7 @@ func (r *Runner) runE(_ *cobra.Command, args []string) error {
Quiet: r.Quiet,
Name: r.Name,
InventoryID: r.InventoryID,
RGFileName: r.RGFile,
Force: r.Force,
}).Run(r.ctx)
if err != nil {
Expand All @@ -129,12 +155,22 @@ type ConfigureInventoryInfo struct {

Name string
InventoryID string
RGFileName string

Force bool
}

// Run updates the inventory info in the package given by the Path.
func (c *ConfigureInventoryInfo) Run(ctx context.Context) error {
// Use ResourceGroup file for inventory logic if the resourcegroup file
// is set directly. For this feature gate, the resourcegroup must be directly set
// through our tests since we are not exposing this through the command surface as a
// flag, currently. When we promote this, the resourcegroup filename can be empty and
// the default filename value will be inferred/used.
if c.RGFileName != "" {
return c.runLiveInitWithRGFile(ctx)
}

const op errors.Op = "cmdliveinit.Run"
pr := printer.FromContextOrDie(ctx)

Expand Down Expand Up @@ -189,6 +225,119 @@ func (c *ConfigureInventoryInfo) Run(ctx context.Context) error {
return nil
}

// func runLiveInitWithRGFile is a modified version of ConfigureInventoryInfo.Run that stores the
// package inventory information in a separate resourcegroup file. The logic for this is branched into
// a separate function to enable feature gating.
func (c *ConfigureInventoryInfo) runLiveInitWithRGFile(ctx context.Context) error {
const op errors.Op = "cmdliveinit.runLiveInitWithRGFile"
pr := printer.FromContextOrDie(ctx)

namespace, err := config.FindNamespace(c.Factory.ToRawKubeConfigLoader(), c.Pkg.UniquePath.String())
if err != nil {
return errors.E(op, c.Pkg.UniquePath, err)
}
namespace = strings.TrimSpace(namespace)
if !c.Quiet {
pr.Printf("initializing ResourceGroup inventory info (namespace: %s)...", namespace)
}

// Autogenerate the name if it is not provided through the flag.
if c.Name == "" {
randomSuffix := common.RandomStr()
c.Name = fmt.Sprintf("%s-%s", defaultInventoryName, randomSuffix)
}

// Finally, create a ResourceGroup containing the inventory information.
err = createRGFile(c.Pkg, &kptfilev1.Inventory{
Namespace: namespace,
Name: c.Name,
InventoryID: c.InventoryID,
}, c.RGFileName, c.Force)
if !c.Quiet {
if err == nil {
pr.Printf("success\n")
} else {
pr.Printf("failed\n")
}
}
if err != nil {
return errors.E(op, c.Pkg.UniquePath, err)
}
// add metrics annotation to package resources to track the usage as the resources
// will be applied using kpt live group
at := attribution.Attributor{PackagePaths: []string{c.Pkg.UniquePath.String()}, CmdGroup: "live"}
at.Process()
return nil
}

// createRGFile fills in the inventory object values into the resourcegroup object and writes to file storage.
func createRGFile(p *pkg.Pkg, inv *kptfilev1.Inventory, filename string, force bool) error {
const op errors.Op = "cmdliveinit.createRGFile"
// Read the resourcegroup object io io.dir
rg, err := p.ReadRGFile(filename)
if err != nil && !goerrors.Is(err, os.ErrNotExist) {
return errors.E(op, p.UniquePath, err)
}

// Read the Kptfile to ensure that inventory information is not in Kptfile either.
kf, err := p.Kptfile()
if err != nil {
return errors.E(op, p.UniquePath, err)
}
// Validate the inventory values don't exist in Kptfile.
isEmpty := kptfileInventoryEmpty(kf.Inventory)
if !isEmpty && !force {
return errors.E(op, p.UniquePath, &InvInKfExistsError{})
}
// Set the Kptfile inventory to be nil if we force write to resourcegroup instead.
kf.Inventory = nil

// Validate the inventory values don't already exist in Resourcegroup.
if rg != nil && !force {
return errors.E(op, p.UniquePath, &InvExistsError{})
}
// Initialize new resourcegroup object, as rg should have been nil.
rg = &rgfilev1alpha1.ResourceGroup{ResourceMeta: rgfilev1alpha1.DefaultMeta}
// // Finally, set the inventory parameters in the ResourceGroup object and write it.
rg.Name = inv.Name
rg.Namespace = inv.Namespace
if inv.InventoryID != "" {
rg.Labels = map[string]string{rgfilev1alpha1.RGInventoryIDLabel: inv.InventoryID}
}
if err := writeRGFile(p.UniquePath.String(), rg, filename); err != nil {
return errors.E(op, p.UniquePath, err)
}

// Rewrite Kptfile without inventory existing Kptfile contains inventory info. This
// is required when a user appends the force flag.
if !isEmpty {
if err := kptfileutil.WriteFile(p.UniquePath.String(), kf); err != nil {
return errors.E(op, p.UniquePath, err)
}
}

return nil
}

// writeRGFile writes a ResourceGroup inventory to local disk.
func writeRGFile(dir string, rg *rgfilev1alpha1.ResourceGroup, filename string) error {
const op errors.Op = "cmdliveinit.writeRGFile"
b, err := yaml.MarshalWithOptions(rg, &yaml.EncoderOptions{SeqIndent: yaml.WideSequenceStyle})
if err != nil {
return err
}
if _, err := os.Stat(filepath.Join(dir, filename)); err != nil && !goerrors.Is(err, os.ErrNotExist) {
return errors.E(op, errors.IO, types.UniquePath(dir), err)
}

// fyi: perm is ignored if the file already exists
err = ioutil.WriteFile(filepath.Join(dir, filename), b, 0600)
if err != nil {
return errors.E(op, errors.IO, types.UniquePath(dir), err)
}
return nil
}

// Run fills in the inventory object values into the Kptfile.
func updateKptfile(p *pkg.Pkg, inv *kptfilev1.Inventory, force bool) error {
const op errors.Op = "cmdliveinit.updateKptfile"
Expand Down
99 changes: 94 additions & 5 deletions internal/cmdliveinit/cmdliveinit_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/GoogleContainerTools/kpt/internal/printer/fake"
"github.com/GoogleContainerTools/kpt/internal/testutil"
kptfilev1 "github.com/GoogleContainerTools/kpt/pkg/api/kptfile/v1"
rgfilev1alpha1 "github.com/GoogleContainerTools/kpt/pkg/api/resourcegroup/v1alpha1"
"github.com/stretchr/testify/assert"
"k8s.io/cli-runtime/pkg/genericclioptions"
cmdtesting "k8s.io/kubectl/pkg/cmd/testing"
Expand Down Expand Up @@ -59,6 +60,14 @@ inventory:

var testTime = time.Unix(5555555, 66666666)

var resourceGroupInventory = `
apiVersion: kpt.dev/v1alpha1
kind: ResourceGroup
metadata:
name: foo
namespace: test-namespace
`

func TestCmd_generateID(t *testing.T) {
testCases := map[string]struct {
namespace string
Expand Down Expand Up @@ -131,6 +140,8 @@ func TestCmd_Run_NoKptfile(t *testing.T) {
func TestCmd_Run(t *testing.T) {
testCases := map[string]struct {
kptfile string
resourcegroup string
rgfilename string
name string
namespace string
inventoryID string
Expand Down Expand Up @@ -162,15 +173,47 @@ func TestCmd_Run(t *testing.T) {
InventoryID: "my-inv-id",
},
},
"Provided values are used with custom resourcegroup filename": {
kptfile: kptFile,
rgfilename: "custom-rg.yaml",
name: "my-pkg",
namespace: "my-ns",
inventoryID: "my-inv-id",
expectedInventory: kptfilev1.Inventory{
Namespace: "my-ns",
Name: "my-pkg",
InventoryID: "my-inv-id",
},
},
"Kptfile with inventory already set is error": {
kptfile: kptFileWithInventory,
name: inventoryName,
namespace: inventoryNamespace,
inventoryID: inventoryID,
force: false,
expectedErrorMsg: "inventory information already set",
},
"ResourceGroup with inventory already set is error": {
kptfile: kptFile,
resourcegroup: resourceGroupInventory,
rgfilename: "resourcegroup.yaml",
name: inventoryName,
namespace: inventoryNamespace,
inventoryID: inventoryID,
force: false,
expectedErrorMsg: "inventory information already set for package",
},
"The force flag allows changing inventory information even if already set": {
"ResourceGroup with inventory and Kptfile with inventory already set is error": {
kptfile: kptFileWithInventory,
resourcegroup: resourceGroupInventory,
rgfilename: "resourcegroup.yaml",
name: inventoryName,
namespace: inventoryNamespace,
inventoryID: inventoryID,
force: false,
expectedErrorMsg: "inventory information already set",
},
"The force flag allows changing inventory information even if already set in Kptfile": {
kptfile: kptFileWithInventory,
name: inventoryName,
namespace: inventoryNamespace,
Expand All @@ -182,6 +225,20 @@ func TestCmd_Run(t *testing.T) {
InventoryID: inventoryID,
},
},
"The force flag allows changing inventory information even if already set in ResourceGroup": {
kptfile: kptFile,
resourcegroup: resourceGroupInventory,
rgfilename: "resourcegroup.yaml",
name: inventoryName,
namespace: inventoryNamespace,
inventoryID: inventoryID,
force: true,
expectedInventory: kptfilev1.Inventory{
Namespace: inventoryNamespace,
Name: inventoryName,
InventoryID: inventoryID,
},
},
}

for tn, tc := range testCases {
Expand All @@ -199,11 +256,21 @@ func TestCmd_Run(t *testing.T) {
t.FailNow()
}

// Create ResourceGroup file if testing the STDIN feature.
if tc.resourcegroup != "" && tc.rgfilename != "" {
err := ioutil.WriteFile(filepath.Join(w.WorkspaceDirectory, tc.rgfilename),
[]byte(tc.resourcegroup), 0600)
if !assert.NoError(t, err) {
t.FailNow()
}
}

revert := testutil.Chdir(t, w.WorkspaceDirectory)
defer revert()

runner := NewRunner(fake.CtxWithDefaultPrinter(), tf, ioStreams)
runner.namespace = tc.namespace
runner.RGFile = tc.rgfilename
args := []string{
"--name", tc.name,
"--inventory-id", tc.inventoryID,
Expand All @@ -224,17 +291,39 @@ func TestCmd_Run(t *testing.T) {
return
}

// Otherwise, validate the kptfile values
// Otherwise, validate the kptfile values and/or resourcegroup values.
var actualInv kptfilev1.Inventory
assert.NoError(t, err)
kf, err := pkg.ReadKptfile(filesys.FileSystemOrOnDisk{}, w.WorkspaceDirectory)
assert.NoError(t, err)
if !assert.NotNil(t, kf.Inventory) {
t.FailNow()

switch tc.rgfilename {
case "":
if !assert.NotNil(t, kf.Inventory) {
t.FailNow()
}
actualInv = *kf.Inventory
default:
// Check resourcegroup file if testing the STDIN feature.
rg, err := pkg.ReadRGFile(w.WorkspaceDirectory, tc.rgfilename)
assert.NoError(t, err)
if !assert.NotNil(t, rg) {
t.FailNow()
}

// Convert resourcegroup inventory back to Kptfile structure so we can share assertion
// logic for Kptfile inventory and ResourceGroup inventory structure.
actualInv = kptfilev1.Inventory{
Name: rg.Name,
Namespace: rg.Namespace,
InventoryID: rg.Labels[rgfilev1alpha1.RGInventoryIDLabel],
}
}
actualInv := *kf.Inventory

expectedInv := tc.expectedInventory
assertInventoryName(t, expectedInv.Name, actualInv.Name)
assert.Equal(t, expectedInv.Namespace, actualInv.Namespace)

if tc.expectAutoGenID {
assertGenInvID(t, actualInv.Name, actualInv.Namespace, actualInv.InventoryID)
} else {
Expand Down

0 comments on commit cc259eb

Please sign in to comment.