teststep apply, assert and errors can now take urls, files or folder (#…

…123)

Signed-off-by: Ken Sipe <kensipe@gmail.com>

pkg/file/files.go

@@ -16,7 +16,7 @@ func ToRuntimeObjects(paths []string) ([]runtime.Object, error) {
 	apply := []runtime.Object{}
 
 	for _, path := range paths {
-		objs, err := testutils.LoadYAML(path)
+		objs, err := testutils.LoadYAMLFromFile(path)
 		if err != nil {
 			return nil, fmt.Errorf("file %q load yaml error", path)
 		}

pkg/http/http.go

@@ -0,0 +1,57 @@
+package http
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	coreHTTP "net/http"
+	"net/url"
+
+	"k8s.io/apimachinery/pkg/runtime"
+
+	testutils "github.com/kudobuilder/kuttl/pkg/test/utils"
+)
+
+// IsURL returns true if string is an URL
+func IsURL(str string) bool {
+	u, err := url.Parse(str)
+	return err == nil && u.Scheme != "" && u.Host != ""
+}
+
+// ToRuntimeObjects takes a url, pulls the file and returns  []runtime.Object
+// url must be a full path to a manifest file.  that file can have multiple runtime objects.
+func ToRuntimeObjects(urlPath string) ([]runtime.Object, error) {
+	apply := []runtime.Object{}
+
+	buf, err := Read(urlPath)
+	if err != nil {
+		return nil, err
+	}
+
+	objs, err := testutils.LoadYAML(urlPath, buf)
+	if err != nil {
+		return nil, fmt.Errorf("url %q load yaml error", urlPath)
+	}
+	apply = append(apply, objs...)
+
+	return apply, nil
+}
+
+// Read returns a buffer for the file at the url
+func Read(urlPath string) (*bytes.Buffer, error) {
+
+	response, err := coreHTTP.Get(urlPath) // nolint:gosec
+	if err != nil {
+		return nil, err
+	}
+	if response != nil {
+		defer response.Body.Close()
+	}
+	var buf bytes.Buffer
+	_, err = io.Copy(&buf, response.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return &buf, nil
+}

pkg/http/http_test.go

@@ -0,0 +1,43 @@
+package http
+
+import (
+	"testing"
+)
+
+func TestIsURL(t *testing.T) {
+
+	tests := []struct {
+		name string
+		path string
+		want bool
+	}{
+		{
+			"path to folder",
+			"/opt/foo",
+			false,
+		},
+		{
+			"path to file",
+			"/opt/foo.txt",
+			false,
+		},
+		{
+			"http to file",
+			"http://kuttl.dev/foo.txt",
+			true,
+		},
+		{
+			"https to file",
+			"https://kuttl.dev/foo.txt",
+			true,
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsURL(tt.path); got != tt.want {
+				t.Errorf("IsURL() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/kuttlctl/cmd/test.go

@@ -80,7 +80,7 @@ For more detailed documentation, visit: https://kudo.dev/docs/testing`,
 
 			// Load the configuration YAML into options.
 			if configPath != "" {
-				objects, err := testutils.LoadYAML(configPath)
+				objects, err := testutils.LoadYAMLFromFile(configPath)
 				if err != nil {
 					return err
 				}

pkg/test/step.go

@@ -17,6 +17,7 @@ import (
 
 	harness "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1"
 	kfile "github.com/kudobuilder/kuttl/pkg/file"
+	"github.com/kudobuilder/kuttl/pkg/http"
 	testutils "github.com/kudobuilder/kuttl/pkg/test/utils"
 )
 
@@ -415,14 +416,14 @@ func (s *Step) String() string {
 	return fmt.Sprintf("%d-%s", s.Index, s.Name)
 }
 
-// LoadYAML loads the resources from a YAML file for a test step:
+// LoadYAMLFromFile loads the resources from a YAML file for a test step:
 // * If the YAML file is called "assert", then it contains objects to
 //   add to the test step's list of assertions.
 // * If the YAML file is called "errors", then it contains objects that,
 //   if seen, mark a test immediately failed.
 // * All other YAML files are considered resources to create.
 func (s *Step) LoadYAML(file string) error {
-	objects, err := testutils.LoadYAML(file)
+	objects, err := testutils.LoadYAMLFromFile(file)
 	if err != nil {
 		return fmt.Errorf("loading %s: %s", file, err)
 	}
@@ -481,37 +482,25 @@ func (s *Step) LoadYAML(file string) error {
 	if s.Step != nil {
 		// process configured step applies
 		for _, applyPath := range s.Step.Apply {
-			paths, err := kfile.FromPath(filepath.Join(s.Dir, applyPath), "*.yaml")
+			apply, err := runtimeObjectsFromPath(s, applyPath)
 			if err != nil {
-				return fmt.Errorf("step %q apply %w", s.Name, err)
-			}
-			apply, err := kfile.ToRuntimeObjects(paths)
-			if err != nil {
-				return fmt.Errorf("step %q apply %w", s.Name, err)
+				return err
 			}
 			applies = append(applies, apply...)
 		}
 		// process configured step asserts
 		for _, assertPath := range s.Step.Assert {
-			paths, err := kfile.FromPath(filepath.Join(s.Dir, assertPath), "*.yaml")
-			if err != nil {
-				return fmt.Errorf("step %q assert %w", s.Name, err)
-			}
-			assert, err := kfile.ToRuntimeObjects(paths)
+			assert, err := runtimeObjectsFromPath(s, assertPath)
 			if err != nil {
-				return fmt.Errorf("step %q assert %w", s.Name, err)
+				return err
 			}
 			asserts = append(asserts, assert...)
 		}
 		// process configured errors
 		for _, errorPath := range s.Step.Error {
-			paths, err := kfile.FromPath(filepath.Join(s.Dir, errorPath), "*.yaml")
+			errObjs, err := runtimeObjectsFromPath(s, errorPath)
 			if err != nil {
-				return fmt.Errorf("step %q error %w", s.Name, err)
-			}
-			errObjs, err := kfile.ToRuntimeObjects(paths)
-			if err != nil {
-				return fmt.Errorf("step %q error %w", s.Name, err)
+				return err
 			}
 			s.Errors = append(s.Errors, errObjs...)
 		}
@@ -521,3 +510,24 @@ func (s *Step) LoadYAML(file string) error {
 	s.Asserts = asserts
 	return nil
 }
+
+func runtimeObjectsFromPath(s *Step, path string) ([]runtime.Object, error) {
+	if http.IsURL(path) {
+		apply, err := http.ToRuntimeObjects(path)
+		if err != nil {
+			return nil, fmt.Errorf("step %q apply %w", s.Name, err)
+		}
+		return apply, nil
+	}
+
+	// it's a directory or file
+	paths, err := kfile.FromPath(filepath.Join(s.Dir, path), "*.yaml")
+	if err != nil {
+		return nil, fmt.Errorf("step %q apply %w", s.Name, err)
+	}
+	apply, err := kfile.ToRuntimeObjects(paths)
+	if err != nil {
+		return nil, fmt.Errorf("step %q apply %w", s.Name, err)
+	}
+	return apply, nil
+}

pkg/test/utils/kubernetes.go

@@ -454,15 +454,19 @@ func MarshalObjectJSON(o runtime.Object, w io.Writer) error {
 	return json.NewSerializer(json.DefaultMetaFactory, nil, nil, false).Encode(copied, w)
 }
 
-// LoadYAML loads all objects from a YAML file.
-func LoadYAML(path string) ([]runtime.Object, error) {
+// LoadYAMLFromFile loads all objects from a YAML file.
+func LoadYAMLFromFile(path string) ([]runtime.Object, error) {
 	opened, err := os.Open(path)
 	if err != nil {
 		return nil, err
 	}
 	defer opened.Close()
 
-	yamlReader := yaml.NewYAMLReader(bufio.NewReader(opened))
+	return LoadYAML(path, opened)
+}
+
+func LoadYAML(path string, r io.Reader) ([]runtime.Object, error) {
+	yamlReader := yaml.NewYAMLReader(bufio.NewReader(r))
 
 	objects := []runtime.Object{}
 
@@ -486,11 +490,18 @@ func LoadYAML(path string) ([]runtime.Object, error) {
 		if err != nil {
 			return nil, fmt.Errorf("error converting unstructured object %s (%s): %w", ResourceID(unstructuredObj), path, err)
 		}
+		// discovered reader will return empty objects if a number of lines are preceding a yaml separator (---)
+		// this detects that, logs and continues
+		if obj.GetObjectKind().GroupVersionKind().Kind == "" {
+			log.Println("object detected with no GVK Kind for path", path)
+		} else {
+			objects = append(objects, obj)
+		}
 
-		objects = append(objects, obj)
 	}
 
 	return objects, nil
+
 }
 
 // MatchesKind returns true if the Kubernetes kind of obj matches any of kinds.
@@ -532,7 +543,7 @@ func InstallManifests(ctx context.Context, client client.Client, dClient discove
 			return nil
 		}
 
-		objs, err := LoadYAML(path)
+		objs, err := LoadYAMLFromFile(path)
 		if err != nil {
 			return err
 		}

pkg/test/utils/kubernetes_test.go

@@ -158,7 +158,7 @@ spec:
 		t.Fatal(err)
 	}
 
-	objs, err := LoadYAML(tmpfile.Name())
+	objs, err := LoadYAMLFromFile(tmpfile.Name())
 	assert.Nil(t, err)
 
 	assert.Equal(t, &unstructured.Unstructured{
@@ -227,7 +227,7 @@ metadata:
 		t.Fatal(err)
 	}
 
-	objs, err := LoadYAML(tmpfile.Name())
+	objs, err := LoadYAMLFromFile(tmpfile.Name())
 	assert.Nil(t, err)
 
 	crd := NewResource("apiextensions.k8s.io/v1beta1", "CustomResourceDefinition", "", "")