Skip to content
This repository has been archived by the owner on Mar 31, 2023. It is now read-only.

Commit

Permalink
Rewrite the manifest parsing to use kyaml & libgitops
Browse files Browse the repository at this point in the history
  • Loading branch information
@luxas @twelho
luxas authored and twelho committed Jul 31, 2020
1 parent bfc899f commit 7c14009
Show file tree
Hide file tree
Showing 2 changed files with 79 additions and 173 deletions.
246 changes: 77 additions & 169 deletions pkg/utilities/manifest/manifest.go
View file
@@ -1,10 +1,9 @@
package manifest

import (
encjson "encoding/json"
"io/ioutil"
"bytes"
"io"
"os"
"reflect"
"strings"

"github.com/pkg/errors"
Expand All @@ -21,201 +20,110 @@ const DefaultNamespace = `weavek8sops`

var DefaultAddonNamespaces = map[string]string{"weave-net": "kube-system"}

//WithNamespace takes in a file or string kubernetes manifest and updates any resources to
//use the namespace specified.
//Returns the updated manifest or an error if there was a problem updating the manifest.
func WithNamespace(fileOrString, namespace string) (string, error) {
content, err := Content(fileOrString)
func WithNamespace(fileOrString, namespace string) ([]byte, error) {
// Set up the readcloser from either a file, or from the given string
var rc io.ReadCloser
if isFile(fileOrString) {
rc = serializer.FromFile(fileOrString)
} else {
rc = serializer.FromBytes([]byte(fileOrString))
}

// Create a FrameReader and FrameWriter, using YAML document separators
// The FrameWriter will write into buf
fr := serializer.NewYAMLFrameReader(rc)
buf := new(bytes.Buffer)
fw := serializer.NewYAMLFrameWriter(buf)

// Read all frames from the FrameReader
frames, err := serializer.ReadFrameList(fr)
if err != nil {
return "", err
return nil, err
}

// If namespace is "", just write all the read frames to buf through the framewriter, and exit
if namespace == "" {
return string(content), nil
}
if err := serializer.WriteFrameList(fw, frames); err != nil {
return nil, err
}

var (
entries []string = strings.Split(string(content), "---\n")
updates []string
)
return buf.Bytes(), nil
}

for _, entry := range entries {
if entry == "" {
continue
}
decode := scheme.Codecs.UniversalDeserializer().Decode
obj, _, err := decode([]byte(entry), nil, nil)
if err != nil {
return "", errors.Wrap(err, "Unable to deserialize entry")
}
err = updateNamespace(&obj, namespace)
if err != nil {
return "", errors.Wrap(err, "unable to update namespace")
}
updated, err := yaml.Marshal(obj)
// Loop through all the frames
for _, frame := range frames {
// Parse the given frame's YAML. JSON also works
obj, err := kyaml.Parse(string(frame))
if err != nil {
return "", errors.Wrap(err, "unable to marshal object")
return nil, nil
}
updates = append(updates, string(updated))
}
return strings.Join(updates, "---\n"), nil
}

func updateNamespace(obj *runtime.Object, namespace string) error {
if err := setNamespace(obj, namespace); err != nil {
return errors.Wrap(err, "Unable to set namespace")
}
if err := setSubjectNamespaceRefs(obj, namespace); err != nil {
return errors.Wrap(err, "Unable to set Subject resource namespace ref")
}
if meta.IsListType(*obj) {
items, err := meta.ExtractList(*obj)
// Get the TypeMeta of the given object
meta, err := obj.GetMeta()
if err != nil {
return errors.Wrap(err, "Unable to extract items from List resource!")
return nil, nil
}

s := json.NewYAMLSerializer(json.DefaultMetaFactory, scheme.Scheme, scheme.Scheme)
for i, item := range items {
hasRawField, err := reflections.HasField(item, "Raw")
if err != nil {
return errors.Wrap(err, "Unable to determine if list item has Raw field")
}
if hasRawField {
r, err := reflections.GetField(item, "Raw")
if err != nil {
return errors.Wrap(err, "Unable to get Raw field to decode")
}
raw := r.([]byte)
o, _, err := s.Decode(raw, nil, nil)
if err != nil {
return errors.Wrap(err, "Unable to decode item's raw field")
}
if err = updateNamespace(&o, namespace); err != nil {
return err
}
items[i] = o
} else {
if err = updateNamespace(&item, namespace); err != nil {
return err
}
items[i] = item
// Use special handling for the v1.List, as we need to traverse each item in the .items list
// Otherwise, just run setNamespaceOnObject for the parsed object
if meta.APIVersion == "v1" && meta.Kind == "List" {
// Visit each item under .items
if err := visitElementsForPath(obj, func(item *kyaml.RNode) error {
// Set namespace on the given item
return setNamespaceOnObject(item, namespace)

}, "items"); err != nil {
return nil, err
}
}
if err = meta.SetList(*obj, items); err != nil {
return errors.Wrap(err, "Unable to set items on List resource")
}
}
return nil
}

func setNamespace(obj *runtime.Object, namespace string) error {
k, err := reflections.GetField(*obj, "Kind")
if err != nil {
return errors.Wrap(err, "Unable to get object kind field")
}
kind := k.(string)
hasField, err := reflections.HasField(*obj, "ObjectMeta")
if err != nil {
return errors.Wrap(err, "unable to determine if object has metadata")
}
if hasField {
v := reflect.ValueOf(*obj).Elem().FieldByName("ObjectMeta")
om := v.Addr().Interface().(*metav1.ObjectMeta)
if kind == "Namespace" {
om.SetName(namespace)
} else {
om.SetNamespace(namespace)
}
if err := reflections.SetField(*obj, "ObjectMeta", *om); err != nil {
return errors.Wrap(err, "unable to update metadata on object")
// Set namespace on the given object
if err := setNamespaceOnObject(obj, namespace); err != nil {
return nil, err
}
}
}
return nil
}

func setSubjectNamespaceRefs(obj *runtime.Object, namespace string) error {
hasSubjects, err := reflections.HasField(*obj, "Subjects")
if err != nil {
return errors.Wrap(err, "Unable to determine if resource has Subject resources")
}
if !hasSubjects {
return nil
}
subjects, err := reflections.GetField(*obj, "Subjects")
if err != nil {
return err
}
switch reflect.TypeOf(subjects).Kind() {
case reflect.Slice:
subs := reflect.ValueOf(subjects)
for i := 0; i < subs.Len(); i++ {
sub := subs.Index(i)
subptr := sub.Addr().Interface()
hasNamespace, err := reflections.HasField(subptr, "Namespace")
if err != nil {
return errors.Wrap(err, "Unable to determine with Subject has a namespace field")
}
if !hasNamespace {
continue
}
if err = reflections.SetField(subptr, "Namespace", namespace); err != nil {
return err
}
subs.Index(i).Set(sub)
// Convert the object to string, and write it to the FrameWriter
str, err := obj.String()
if err != nil {
return nil, err
}
if err = reflections.SetField(*obj, "Subjects", subs.Interface()); err != nil {
return errors.Wrap(err, "Unable to set namespace references")
if _, err := fw.Write([]byte(str)); err != nil {
return nil, err
}
default:
return errors.New("Subjects should be a slice but wasn't")
}
return nil

return buf.Bytes(), nil
}

//Content returns a byte slice representing the yaml manifest retrieved from a passed in file or string
func Content(fileOrString string) ([]byte, error) {
if !isFile(fileOrString) {
return convertJSONToYAMLIfNecessary(fileOrString)
}
file, err := os.Open(fileOrString)
func setNamespaceOnObject(obj *kyaml.RNode, namespace string) error {
// Lookup and create .metadata (if it doesn't exist), and set its
// namespace field to the desired value
err := obj.PipeE(
kyaml.LookupCreate(kyaml.MappingNode, "metadata"),
setNamespaceFilter(namespace),
)
if err != nil {
return nil, errors.Wrap(err, "failed to open manifest")
return err
}

defer file.Close()

content, err := ioutil.ReadAll(file)
if err != nil {
return nil, errors.Wrap(err, "failed to read manifest")
}
return convertJSONToYAMLIfNecessary(string(content))
// Visit .subjects (if it exists), and traverse its elements, setting
// the namespace field on each item
return visitElementsForPath(obj, func(node *kyaml.RNode) error {
return node.PipeE(setNamespaceFilter(namespace))
}, "subjects")
}

func convertJSONToYAMLIfNecessary(yamlOrJson string) ([]byte, error) {
if isJSON(yamlOrJson) {
yaml, err := jsonToYaml(yamlOrJson)
if err != nil {
return nil, err
}
return yaml, nil
func visitElementsForPath(obj *kyaml.RNode, fn func(node *kyaml.RNode) error, paths ...string) error {
list, err := obj.Pipe(kyaml.Lookup(paths...))
if err != nil {
return err
}
return []byte(yamlOrJson), nil
}

func isJSON(s string) bool {
var js string
return encjson.Unmarshal([]byte(s), &js) == nil
return list.VisitElements(fn)
}

func jsonToYaml(content string) ([]byte, error) {
cbytes := []byte(content)
bytes, err := yaml.JSONToYAML(cbytes)
if err != nil {
return nil, err
}
if len(bytes) == 0 {
return nil, errors.New("Could not convert json to yaml")
}
return bytes, nil
func setNamespaceFilter(ns string) kyaml.FieldSetter {
return kyaml.SetField("namespace", kyaml.NewScalarRNode(ns))
}

func isFile(fileOrString string) bool {
Expand Down
6 changes: 2 additions & 4 deletions pkg/utilities/manifest/manifest_test.go
View file
Expand Up @@ -250,8 +250,6 @@ var nstests = []struct {
}

func TestManifestWithNamespace(t *testing.T) {
assert.NoError(t, clusterv1.AddToScheme(scheme.Scheme))
assert.NoError(t, existinginfrav1.AddToScheme(scheme.Scheme))
for _, tt := range nstests {
t.Run(tt.name, func(t *testing.T) {
fname := createFile(t, tt.content, tt.fileName).Name()
Expand All @@ -260,8 +258,8 @@ func TestManifestWithNamespace(t *testing.T) {

updated, err := WithNamespace(fname, newNamespace)
assert.NoError(t, err)
assert.NotEqual(t, tt.content, updated)
assert.Contains(t, updated, newNamespace)
assert.NotEqual(t, tt.content, string(updated))
assert.Contains(t, string(updated), newNamespace)
})
}
}

0 comments on commit 7c14009

Please sign in to comment.