Skip to content

Commit

Permalink
Autosetters during pkg update
Browse files Browse the repository at this point in the history
  • Loading branch information
@phanimarupaka
phanimarupaka committed Sep 25, 2020
1 parent 5f97bc4 commit 36cc247
Show file tree
Hide file tree
Showing 7 changed files with 157 additions and 106 deletions.
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,4 @@ require (
sigs.k8s.io/kustomize/cmd/config v0.8.2-0.20200924195921-3a5951563dec
sigs.k8s.io/kustomize/kyaml v0.8.2-0.20200924195921-3a5951563dec
)

38 changes: 0 additions & 38 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -609,53 +609,15 @@ modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
sigs.k8s.io/cli-utils v0.19.2/go.mod h1:ulIQPERYwkYksNriRknJRbGECDR/ZZROpkiThlXBtB4=
sigs.k8s.io/cli-utils v0.19.3-0.20200901170551-5a58b8941377 h1:o5aB6N3nszCNdQVj1MAlCLLW6nndJ4QYYeodWX+Dm0U=
sigs.k8s.io/cli-utils v0.19.3-0.20200901170551-5a58b8941377/go.mod h1:ulIQPERYwkYksNriRknJRbGECDR/ZZROpkiThlXBtB4=
sigs.k8s.io/cli-utils v0.20.0 h1:45nMRNJUZF+ceQ531+ilgZOVMYzP/x3Lk4dhcivuTm4=
sigs.k8s.io/cli-utils v0.20.0/go.mod h1:ulIQPERYwkYksNriRknJRbGECDR/ZZROpkiThlXBtB4=
sigs.k8s.io/cli-utils v0.20.1 h1:S3IM4Rmely9oWNAeytx46PhngTwdRxsJ4ixtoGWPc3o=
sigs.k8s.io/cli-utils v0.20.1/go.mod h1:fQkEMmLXduMNmUf1+dgQ5GRJa4OrrLMw3qzAN8YXAU8=
sigs.k8s.io/cli-utils v0.20.2 h1:jNMu4ExtvFXlmKqZMDJqySqK55vGtVRqoLht6eIMffw=
sigs.k8s.io/cli-utils v0.20.2/go.mod h1:uT5cVxMrOoRplL8umtrJx5R51ZWsIKD7lQfPtot80uA=
sigs.k8s.io/cli-utils v0.20.3-0.20200922200010-e382589cbb02 h1:0qGR+FjSQ+h3atni5j2MgGpE34YjA5kfr6KUaf6BhrQ=
sigs.k8s.io/cli-utils v0.20.3-0.20200922200010-e382589cbb02/go.mod h1:uT5cVxMrOoRplL8umtrJx5R51ZWsIKD7lQfPtot80uA=
sigs.k8s.io/controller-runtime v0.4.0 h1:wATM6/m+3w8lj8FXNaO6Fs/rq/vqoOjO1Q116Z9NPsg=
sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns=
sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0=
sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU=
sigs.k8s.io/kustomize/cmd/config v0.7.1-0.20200903155941-0f49fef5ed58 h1:sc+6pqiiR3+MlOTcKrqo20j9dQiiXRDiqd0SUfdXtcU=
sigs.k8s.io/kustomize/cmd/config v0.7.1-0.20200903155941-0f49fef5ed58/go.mod h1:Hf7Qf/NCo3PEglHxO/2YkV2L9J5mL5eJTiNqzb+V5ww=
sigs.k8s.io/kustomize/cmd/config v0.7.1-0.20200910020544-badc1177d9e5 h1:ocuBK8I8bAKgdxQThYfCIe05a7RpqYAA/etUprZ9gLg=
sigs.k8s.io/kustomize/cmd/config v0.7.1-0.20200910020544-badc1177d9e5/go.mod h1:Hf7Qf/NCo3PEglHxO/2YkV2L9J5mL5eJTiNqzb+V5ww=
sigs.k8s.io/kustomize/cmd/config v0.7.1-0.20200914180048-6a0a909e7315 h1:kmPh9PctbqHRbE8nps5iFOsW4Ybm/H4dqdVi34hrWK0=
sigs.k8s.io/kustomize/cmd/config v0.7.1-0.20200914180048-6a0a909e7315/go.mod h1:Hf7Qf/NCo3PEglHxO/2YkV2L9J5mL5eJTiNqzb+V5ww=
sigs.k8s.io/kustomize/cmd/config v0.7.1-0.20200914202724-1c6481d011df h1:E+ngf8WXySmILexnoiYyef9hSZxKPTvvku2uUhbOJGw=
sigs.k8s.io/kustomize/cmd/config v0.7.1-0.20200914202724-1c6481d011df/go.mod h1:w6fhpKsRmQlOkSS2be/3TPTDZzMsWQ5Kj9Av1lDPXFE=
sigs.k8s.io/kustomize/cmd/config v0.7.1-0.20200915164007-ad7fed061e2f h1:mk5lAFWtjaxOKG78ei5A39Av0hG/Vy5wBZn1iQVT5cs=
sigs.k8s.io/kustomize/cmd/config v0.7.1-0.20200915164007-ad7fed061e2f/go.mod h1:w6fhpKsRmQlOkSS2be/3TPTDZzMsWQ5Kj9Av1lDPXFE=
sigs.k8s.io/kustomize/cmd/config v0.8.0 h1:zrLcS03Wk8+azb3cmKuPTkIFBFg9BP0aZ1GYzEPIgFQ=
sigs.k8s.io/kustomize/cmd/config v0.8.0/go.mod h1:3vnx3SFRfYrPJfzG8q/fHpCvjjF7ksZSt7t6nNe5qkk=
sigs.k8s.io/kustomize/cmd/config v0.8.2-0.20200922204209-cf38166bd62c h1:TYKkSYTSPSum0iB/nGf7H9uHWn8sSkJIlRUd7ZllqXc=
sigs.k8s.io/kustomize/cmd/config v0.8.2-0.20200922204209-cf38166bd62c/go.mod h1:UZ6TTvgftENXmRS3YWgxLkLakI9ilzTdIFyK/usXUyo=
sigs.k8s.io/kustomize/cmd/config v0.8.2-0.20200924195921-3a5951563dec h1://OdIqwosbLerHd6Zk4/Y+BWSuFjlXmaEUmJU8gxyyM=
sigs.k8s.io/kustomize/cmd/config v0.8.2-0.20200924195921-3a5951563dec/go.mod h1:PlMvypgZFfP2rQKS5tOpYHnRb35tR1wJo7C+frqudWU=
sigs.k8s.io/kustomize/kyaml v0.7.1/go.mod h1:ne3F9JPhW2wrVaLslxBsEe6MQJQ9YK5rUutrdhBWXwI=
sigs.k8s.io/kustomize/kyaml v0.7.2-0.20200903155941-0f49fef5ed58 h1:5Yjha1M1CQfaCl4SlUC/9lPJyQfy9ljgLaY19Ja+wus=
sigs.k8s.io/kustomize/kyaml v0.7.2-0.20200903155941-0f49fef5ed58/go.mod h1:UTm64bSWVdBUA8EQoYCxVOaBQxUdIOr5LKWxA4GNbkw=
sigs.k8s.io/kustomize/kyaml v0.7.2-0.20200910020544-badc1177d9e5 h1:mTYqbcQbs84fxs+TYTtp6D5x/K7zo1p0OC6eIW+9hNI=
sigs.k8s.io/kustomize/kyaml v0.7.2-0.20200910020544-badc1177d9e5/go.mod h1:UTm64bSWVdBUA8EQoYCxVOaBQxUdIOr5LKWxA4GNbkw=
sigs.k8s.io/kustomize/kyaml v0.7.2-0.20200914180048-6a0a909e7315 h1:D+cp/iWpVHy8+NxBpyJ4PuQ0S9w6G6YNuXz3BxCN6w4=
sigs.k8s.io/kustomize/kyaml v0.7.2-0.20200914180048-6a0a909e7315/go.mod h1:UTm64bSWVdBUA8EQoYCxVOaBQxUdIOr5LKWxA4GNbkw=
sigs.k8s.io/kustomize/kyaml v0.7.2-0.20200914202724-1c6481d011df h1:mWY8Jj8q782kuOMTisAxK9lfezZM7Im3Fna17aGrWBY=
sigs.k8s.io/kustomize/kyaml v0.7.2-0.20200914202724-1c6481d011df/go.mod h1:UTm64bSWVdBUA8EQoYCxVOaBQxUdIOr5LKWxA4GNbkw=
sigs.k8s.io/kustomize/kyaml v0.7.2-0.20200915164007-ad7fed061e2f h1:vLk/yEunT6nBTybPdaoK8EcHiJZANIVsa4avZd79Xm0=
sigs.k8s.io/kustomize/kyaml v0.7.2-0.20200915164007-ad7fed061e2f/go.mod h1:UTm64bSWVdBUA8EQoYCxVOaBQxUdIOr5LKWxA4GNbkw=
sigs.k8s.io/kustomize/kyaml v0.8.0 h1:/MqPML99XAm2pbrD/eTpePh5rnU5bpnuTPqb29LpSz4=
sigs.k8s.io/kustomize/kyaml v0.8.0/go.mod h1:UTm64bSWVdBUA8EQoYCxVOaBQxUdIOr5LKWxA4GNbkw=
sigs.k8s.io/kustomize/kyaml v0.8.1/go.mod h1:UTm64bSWVdBUA8EQoYCxVOaBQxUdIOr5LKWxA4GNbkw=
sigs.k8s.io/kustomize/kyaml v0.8.2-0.20200922204209-cf38166bd62c h1:jvC8RRmfCZ3RzJ71tySDQJYldpPMioCyxw0IYOzd3Xg=
sigs.k8s.io/kustomize/kyaml v0.8.2-0.20200922204209-cf38166bd62c/go.mod h1:UTm64bSWVdBUA8EQoYCxVOaBQxUdIOr5LKWxA4GNbkw=
sigs.k8s.io/kustomize/kyaml v0.8.2-0.20200924195921-3a5951563dec h1:Nz39T2Tnj6mRG0Rhfz8YJHxrtMtPTQ1F9roBPUPMles=
sigs.k8s.io/kustomize/kyaml v0.8.2-0.20200924195921-3a5951563dec/go.mod h1:UTm64bSWVdBUA8EQoYCxVOaBQxUdIOr5LKWxA4GNbkw=
sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
Expand Down
2 changes: 1 addition & 1 deletion internal/cmdget/cmdget.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func (r *Runner) runE(c *cobra.Command, args []string) error {
}

if r.AutoSet {
if err := setters.PerformSetters(r.Get.Destination); err != nil {
if err := setters.PerformAutoSetters(r.Get.Destination); err != nil {
return err
}
}
Expand Down
163 changes: 97 additions & 66 deletions internal/util/setters/setters.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,50 +37,22 @@ const (
GcloudProjectNumber = "gcloud.project.projectNumber"
)

func PerformSetters(path string) error {
// auto-fill setters from the environment
for i := range os.Environ() {
e := os.Environ()[i]
if !strings.HasPrefix(e, "KPT_SET_") {
continue
}
parts := strings.SplitN(e, "=", 2)
if len(parts) < 2 {
continue
}
k, v := strings.TrimPrefix(parts[0], "KPT_SET_"), parts[1]
k = strings.ToLower(k)

rw := &kio.LocalPackageReadWriter{
PackagePath: path,
KeepReaderAnnotations: false,
IncludeSubpackages: true,
}

setter := &setters.PerformSetters{Name: k, Value: v, SetBy: "kpt"}
err := kio.Pipeline{
Inputs: []kio.Reader{rw},
Filters: []kio.Filter{setter},
Outputs: []kio.Writer{rw},
}.Execute()

if err != nil {
return err
}

setter2 := &settersutil.FieldSetter{Name: k, Value: v, SetBy: "kpt", ResourcesPath: path, OpenAPIPath: filepath.Join(path, kptfile.KptFileName)}
err = kio.Pipeline{
Inputs: []kio.Reader{rw},
Filters: []kio.Filter{setter2},
Outputs: []kio.Writer{rw},
}.Execute()
func PerformAutoSetters(path string) error {
// auto-fill setters from environment
if err := SetEnvAutoSetters(path); err != nil {
return err
}

if err != nil {
return err
}
// auto-fill setters from gcloud config
if err := SetGcloudAutoSetters(path); err != nil {
return err
}

// auto-fill setters from gcloud
return nil
}

// SetGcloudAutoSetters auto-fills setters from gcloud config
func SetGcloudAutoSetters(path string) error {
gcloudConfig := []string{"compute.region", "compute.zone", "core.project"}
for _, c := range gcloudConfig {
gcloudCmd := exec.Command("gcloud",
Expand All @@ -106,34 +78,9 @@ func PerformSetters(path string) error {
return err
}
}

return nil
}

var GetProjectNumberFromProjectID = func(projectID string) (string, error) {
gcloudCmd := exec.Command("gcloud",
"projects", "describe", projectID, "--format", "value(projectNumber)")
b, err := gcloudCmd.Output()
if err != nil {
return "", errors.Wrapf(err, "failed to get project number for %s, please verify gcloud "+
"credentials are valid and try again", projectID)
}
return strings.TrimSpace(string(b)), nil
}

// DefExists returns true if the setterName exists in Kptfile definitions
func DefExists(resourcePath, setterName string) bool {
if err := openapi.AddSchemaFromFile(filepath.Join(resourcePath, kptfile.KptFileName)); err != nil {
return false
}
ref, err := spec.NewRef(fieldmeta.DefinitionsPrefix + fieldmeta.SetterDefinitionPrefix + setterName)
if err != nil {
return false
}
setter, _ := openapi.Resolve(&ref)
return setter != nil
}

// SetV1AutoSetter sets the input auto setter recursively in all the sub-packages of root
// Sets GcloudProjectNumber as well, if input setter is GcloudProject
func SetV1AutoSetter(name, value, path string) error {
Expand Down Expand Up @@ -204,6 +151,7 @@ func SetV2AutoSetter(name, value, root string) error {
rw := &kio.LocalPackageReadWriter{
PackagePath: resourcesPath,
KeepReaderAnnotations: false,
PackageFileName: kptfile.KptFileName,
}
err = kio.Pipeline{
Inputs: []kio.Reader{rw},
Expand Down Expand Up @@ -243,3 +191,86 @@ func SetV2AutoSetter(name, value, root string) error {
}
return nil
}

var GetProjectNumberFromProjectID = func(projectID string) (string, error) {
gcloudCmd := exec.Command("gcloud",
"projects", "describe", projectID, "--format", "value(projectNumber)")
b, err := gcloudCmd.Output()
if err != nil {
return "", errors.Wrapf(err, "failed to get project number for %s, please verify gcloud "+
"credentials are valid and try again", projectID)
}
return strings.TrimSpace(string(b)), nil
}

// SetEnvAutoSetters auto-fills setters from the environment
func SetEnvAutoSetters(root string) error {
resourcePackagesPaths, err := pathutil.DirsWithFile(root, kptfile.KptFileName, true)
if err != nil {
return err
}
envVariables := environmentVariables()
for i := range envVariables {
e := envVariables[i]
if !strings.HasPrefix(e, "KPT_SET_") {
continue
}
parts := strings.SplitN(e, "=", 2)
if len(parts) < 2 {
continue
}
k, v := strings.TrimPrefix(parts[0], "KPT_SET_"), parts[1]

for _, resourcesPath := range resourcePackagesPaths {
rw := &kio.LocalPackageReadWriter{
PackagePath: resourcesPath,
KeepReaderAnnotations: false,
PackageFileName: kptfile.KptFileName,
}

setter := &setters.PerformSetters{Name: k, Value: v, SetBy: "kpt"}
err := kio.Pipeline{
Inputs: []kio.Reader{rw},
Filters: []kio.Filter{setter},
Outputs: []kio.Writer{rw},
}.Execute()

if err != nil {
return err
}

setter2 := &settersutil.FieldSetter{
Name: k,
Value: v,
SetBy: "kpt",
ResourcesPath: resourcesPath,
OpenAPIPath: filepath.Join(resourcesPath, kptfile.KptFileName),
}
err = kio.Pipeline{
Inputs: []kio.Reader{rw},
Filters: []kio.Filter{setter2},
Outputs: []kio.Writer{rw},
}.Execute()

if err != nil {
return err
}
}
}
return nil
}

var environmentVariables = os.Environ

// DefExists returns true if the setterName exists in Kptfile definitions
func DefExists(resourcePath, setterName string) bool {
if err := openapi.AddSchemaFromFile(filepath.Join(resourcePath, kptfile.KptFileName)); err != nil {
return false
}
ref, err := spec.NewRef(fieldmeta.DefinitionsPrefix + fieldmeta.SetterDefinitionPrefix + setterName)
if err != nil {
return false
}
setter, _ := openapi.Resolve(&ref)
return setter != nil
}
51 changes: 51 additions & 0 deletions internal/util/setters/setters_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,3 +135,54 @@ func TestSetV2AutoSetter(t *testing.T) {
})
}
}

func TestEnvironmentSetters(t *testing.T) {
var tests = []struct {
name string
envVariables []string
dataset string
expectedDataset string
}{
{
name: "autoset-recurse-subpackages",
dataset: "dataset-with-autosetters",
envVariables: []string{"KPT_SET_gcloud.core.project=my-project",
"KPT_SET_gcloud.project.projectNumber=1234"},
expectedDataset: "dataset-with-autosetters-set",
},
}
for i := range tests {
test := tests[i]
t.Run(test.name, func(t *testing.T) {
// reset the openAPI afterward
openapi.ResetOpenAPI()
defer openapi.ResetOpenAPI()
testDataDir := filepath.Join("../../", "testutil", "testdata")
sourceDir := filepath.Join(testDataDir, test.dataset)
expectedDir := filepath.Join(testDataDir, test.expectedDataset)
baseDir, err := ioutil.TempDir("", "")
if !assert.NoError(t, err) {
t.FailNow()
}
err = copyutil.CopyDir(sourceDir, baseDir)
if !assert.NoError(t, err) {
t.FailNow()
}
defer os.RemoveAll(baseDir)
environmentVariables = func() []string {
return test.envVariables
}
err = SetEnvAutoSetters(baseDir)
if !assert.NoError(t, err) {
t.FailNow()
}
diff, err := copyutil.Diff(baseDir, expectedDir)
if !assert.NoError(t, err) {
t.FailNow()
}
if !assert.Equal(t, 0, diff.Len()) {
t.FailNow()
}
})
}
}
2 changes: 1 addition & 1 deletion internal/util/sync/sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func (c Command) Run() error {
}
path := filepath.Join(c.Dir, dep.Name)
if dep.AutoSet {
if err := setters.PerformSetters(path); err != nil {
if err := setters.PerformAutoSetters(path); err != nil {
return err
}
}
Expand Down
6 changes: 6 additions & 0 deletions internal/util/update/resource-merge.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (

"github.com/GoogleContainerTools/kpt/internal/util/get"
"github.com/GoogleContainerTools/kpt/internal/util/git"
"github.com/GoogleContainerTools/kpt/internal/util/setters"
"github.com/GoogleContainerTools/kpt/pkg/kptfile"
"github.com/GoogleContainerTools/kpt/pkg/kptfile/kptfileutil"
"sigs.k8s.io/kustomize/kyaml/copyutil"
Expand Down Expand Up @@ -58,6 +59,11 @@ func (u ResourceMergeUpdater) Update(options UpdateOptions) error {
}
defer os.RemoveAll(updated.AbsPath())

err := setters.PerformAutoSetters(updated.AbsPath())
if err != nil {
return err
}

kf, err := u.updatedKptfile(updated.AbsPath(), original.AbsPath(), options)
if err != nil {
return err
Expand Down

0 comments on commit 36cc247

Please sign in to comment.