add 'categories' to search, kind update to 0.9.0

.github/workflows/compile-test.yaml

@@ -20,7 +20,7 @@ jobs:
         sudo ufw allow 2200:2300/tcp
         sudo ufw enable
         sudo ufw status verbose
-        GO111MODULE=on go get sigs.k8s.io/kind@v0.7.0
+        GO111MODULE=on go get sigs.k8s.io/kind@v0.9.0
         GO111MODULE=on go test -timeout 600s -v -p 1 ./...
 
     - name: Run gofmt

go.mod

@@ -3,21 +3,27 @@ module github.com/vmware-tanzu/crash-diagnostics
 go 1.15
 
 require (
-	github.com/imdario/mergo v0.3.7 // indirect
-	github.com/onsi/ginkgo v1.10.1
+	github.com/go-logr/logr v0.2.1 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/googleapis/gnostic v0.5.2 // indirect
+	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/onsi/ginkgo v1.11.0
 	github.com/onsi/gomega v1.7.0
-	github.com/pkg/errors v0.8.0
-	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/cobra v0.0.5
+	github.com/pkg/errors v0.9.1
+	github.com/sirupsen/logrus v1.7.0
+	github.com/spf13/cobra v1.0.0
 	github.com/vladimirvivien/echo v0.0.1-alpha.6
-	go.starlark.net v0.0.0-20200615180055-61b64bc45990
-	golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8
-	golang.org/x/sys v0.0.0-20200113162924-86b910548bc1 // indirect
-	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
-	gopkg.in/yaml.v2 v2.2.7 // indirect
-	k8s.io/api v0.0.0-20190828114745-198695d0603e
-	k8s.io/apimachinery v0.17.0
-	k8s.io/cli-runtime v0.0.0-20190828120509-9a5048624be8
-	k8s.io/client-go v0.0.0-20190828114957-b4d94f01600c
-	k8s.io/utils v0.0.0-20190809000727-6c36bc71fc4a // indirect
+	go.starlark.net v0.0.0-20201006213952-227f4aabceb5
+	golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee
+	golang.org/x/net v0.0.0-20201010224723-4f7140c49acb // indirect
+	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
+	golang.org/x/sys v0.0.0-20201013132646-2da7054afaeb // indirect
+	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	k8s.io/api v0.19.2
+	k8s.io/apimachinery v0.19.2
+	k8s.io/cli-runtime v0.19.0
+	k8s.io/client-go v0.19.0
+	k8s.io/klog/v2 v2.3.0 // indirect
+	k8s.io/utils v0.0.0-20201005171033-6301aaf42dc7 // indirect
 )

k8s/client.go

@@ -4,6 +4,7 @@
 package k8s
 
 import (
+	"context"
 	"strings"
 
 	"github.com/sirupsen/logrus"
@@ -66,8 +67,9 @@ func New(kubeconfig string) (*Client, error) {
 	return &Client{Client: client, Disco: disco, CoreRest: restc}, nil
 }
 
-func (k8sc *Client) Search(params SearchParams) ([]SearchResult, error) {
-	return k8sc._search(strings.Join(params.Groups, " "),
+func (k8sc *Client) Search(ctx context.Context, params SearchParams) ([]SearchResult, error) {
+	return k8sc._search(ctx, strings.Join(params.Groups, " "),
+		strings.Join(params.Categories, " "),
 		strings.Join(params.Kinds, " "),
 		strings.Join(params.Namespaces, " "),
 		strings.Join(params.Versions, " "),
@@ -82,18 +84,20 @@ func (k8sc *Client) Search(params SearchParams) ([]SearchResult, error) {
 // 3) kinds will match resource.Kind or resource.Name
 // 4) All search params are passed as comma- or space-separated sets that are matched using OR (i.e. kinds=pods services
 //    will match resouces of type pods or services)
-func (k8sc *Client) _search(groups, kinds, namespaces, versions, names, labels, containers string) ([]SearchResult, error) {
+func (k8sc *Client) _search(ctx context.Context, groups, categories, kinds, namespaces, versions, names, labels, containers string) ([]SearchResult, error) {
+
 	// normalize params
 	groups = strings.ToLower(groups)
+	categories = strings.ToLower(categories)
 	kinds = strings.ToLower(kinds)
 	namespaces = strings.ToLower(namespaces)
 	versions = strings.ToLower(versions)
 	labels = strings.ToLower(labels)
 	containers = strings.ToLower(containers)
 
 	logrus.Debugf(
-		"Search filters groups:[%v]; kinds:[%v]; namespaces:[%v]; versions:[%v]; names:[%v]; labels:[%v] containers:[%s]",
-		groups, kinds, namespaces, versions, names, labels, containers,
+		"Search filters groups:[%v]; categories:[%v]; kinds:[%v]; namespaces:[%v]; versions:[%v]; names:[%v]; labels:[%v] containers:[%s]",
+		groups, categories, kinds, namespaces, versions, names, labels, containers,
 	)
 
 	grpList, err := k8sc.Disco.ServerGroups()
@@ -103,7 +107,7 @@ func (k8sc *Client) _search(groups, kinds, namespaces, versions, names, labels,
 
 	// if namespace filters not provided, assume all namespaces
 	if len(namespaces) == 0 {
-		nsNames, err := getNamespaces(k8sc)
+		nsNames, err := getNamespaces(ctx, k8sc)
 		if err != nil {
 			return nil, err
 		}
@@ -140,12 +144,16 @@ func (k8sc *Client) _search(groups, kinds, namespaces, versions, names, labels,
 				continue
 			}
 
-			// filter by resource kind
+			// filter by resource kind and categories
 			for _, res := range resources.APIResources {
 				if len(kinds) > 0 && !strings.Contains(kinds, strings.ToLower(res.Kind)) {
 					continue
 				}
 
+				if len(categories) > 0 && !sliceContains(splitParamList(categories), res.Categories...) {
+					continue
+				}
+
 				gvr := schema.GroupVersionResource{
 					Group:    toLegacyGrpName(grpName),
 					Version:  discoGV.Version,
@@ -164,7 +172,7 @@ func (k8sc *Client) _search(groups, kinds, namespaces, versions, names, labels,
 				if res.Namespaced {
 					for _, ns := range splitParamList(namespaces) {
 						logrus.Debugf("Searching for %s in namespace %s [GroupRes: %v]", res.Name, ns, gvr)
-						list, err := k8sc.Client.Resource(gvr).Namespace(ns).List(listOptions)
+						list, err := k8sc.Client.Resource(gvr).Namespace(ns).List(ctx, listOptions)
 						if err != nil {
 							logrus.Debugf(
 								"WARN: K8s.Search failed to get %s in %s [GroupRes: %s][labels: %v]: %s",
@@ -186,7 +194,7 @@ func (k8sc *Client) _search(groups, kinds, namespaces, versions, names, labels,
 					}
 				} else {
 					logrus.Debugf("Searching for resource %s (non-namespaced)", res.Name)
-					list, err := k8sc.Client.Resource(gvr).List(listOptions)
+					list, err := k8sc.Client.Resource(gvr).List(ctx, listOptions)
 					if err != nil {
 						logrus.Debugf(
 							"WARN: K8s.Search failed to get %s: [GroupRes: %s] [labels: %v]: %s",
@@ -330,13 +338,13 @@ func getPodContainers(podItem unstructured.Unstructured) []interface{} {
 }
 
 // getNamespaces collect all available namespaces in cluster
-func getNamespaces(k8sc *Client) ([]string, error) {
+func getNamespaces(ctx context.Context, k8sc *Client) ([]string, error) {
 	gvr := schema.GroupVersionResource{
 		Group:    "",
 		Version:  "v1",
 		Resource: "namespaces",
 	}
-	objList, err := k8sc.Client.Resource(gvr).List(metav1.ListOptions{})
+	objList, err := k8sc.Client.Resource(gvr).List(ctx, metav1.ListOptions{})
 
 	if err != nil {
 		return nil, err
@@ -349,3 +357,16 @@ func getNamespaces(k8sc *Client) ([]string, error) {
 
 	return names, nil
 }
+
+// sliceContains check if atleast one of the values is present in the slice
+func sliceContains(slice []string, values ...string) bool {
+	for _, s := range slice {
+		for _, v := range values {
+			if strings.EqualFold(strings.TrimSpace(s), strings.TrimSpace(v)) {
+				return true
+			}
+		}
+	}
+
+	return false
+}

k8s/container_logger.go

@@ -1,6 +1,7 @@
 package k8s
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"os"
@@ -27,10 +28,10 @@ func NewContainerLogger(namespace, podName string, container corev1.Container) C
 	}
 }
 
-func (c ContainerLogsImpl) Fetch(restApi rest.Interface) (io.ReadCloser, error) {
+func (c ContainerLogsImpl) Fetch(ctx context.Context, restApi rest.Interface) (io.ReadCloser, error) {
 	opts := &corev1.PodLogOptions{Container: c.container.Name}
 	req := restApi.Get().Namespace(c.namespace).Name(c.podName).Resource("pods").SubResource("log").VersionedParams(opts, scheme.ParameterCodec)
-	stream, err := req.Stream()
+	stream, err := req.Stream(ctx)
 	if err != nil {
 		err = errors.Wrap(err, "failed to create container log stream")
 	}

k8s/k8s.go

@@ -1,6 +1,7 @@
 package k8s
 
 import (
+	"context"
 	"fmt"
 	"io"
 
@@ -10,7 +11,7 @@ import (
 const BaseDirname = "kubecapture"
 
 type Container interface {
-	Fetch(rest.Interface) (io.ReadCloser, error)
+	Fetch(context.Context, rest.Interface) (io.ReadCloser, error)
 	Write(io.ReadCloser, string) error
 }
 

k8s/nodes.go

@@ -4,18 +4,20 @@
 package k8s
 
 import (
+	"context"
+
 	"github.com/pkg/errors"
 	coreV1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
-func GetNodeAddresses(kubeconfigPath string, labels, names []string) ([]string, error) {
+func GetNodeAddresses(ctx context.Context, kubeconfigPath string, labels, names []string) ([]string, error) {
 	client, err := New(kubeconfigPath)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not initialize search client")
 	}
 
-	nodes, err := getNodes(client, names, labels)
+	nodes, err := getNodes(ctx, client, names, labels)
 	if err != nil {
 		return nil, errors.Wrapf(err, "could not fetch nodes")
 	}
@@ -27,8 +29,8 @@ func GetNodeAddresses(kubeconfigPath string, labels, names []string) ([]string,
 	return nodeIps, nil
 }
 
-func getNodes(k8sc *Client, names, labels []string) ([]*coreV1.Node, error) {
-	nodeResults, err := k8sc.Search(SearchParams{
+func getNodes(ctx context.Context, k8sc *Client, names, labels []string) ([]*coreV1.Node, error) {
+	nodeResults, err := k8sc.Search(ctx, SearchParams{
 		Groups: []string{"core"},
 		Kinds:  []string{"nodes"},
 		Names:  names,

k8s/result_writer.go

@@ -1,6 +1,7 @@
 package k8s
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -33,7 +34,7 @@ func (w *ResultWriter) GetResultDir() string {
 	return w.workdir
 }
 
-func (w *ResultWriter) Write(searchResults []SearchResult) error {
+func (w *ResultWriter) Write(ctx context.Context, searchResults []SearchResult) error {
 	if len(searchResults) == 0 {
 		return fmt.Errorf("cannot write empty (or nil) search result")
 	}
@@ -64,7 +65,7 @@ func (w *ResultWriter) Write(searchResults []SearchResult) error {
 					return err
 				}
 				for _, containerLogger := range containers {
-					reader, err := containerLogger.Fetch(w.restApi)
+					reader, err := containerLogger.Fetch(ctx, w.restApi)
 					if err != nil {
 						return err
 					}

k8s/search_params.go

@@ -11,6 +11,7 @@ import (
 
 type SearchParams struct {
 	Groups     []string
+	Categories []string
 	Kinds      []string
 	Namespaces []string
 	Versions   []string

starlark/capa_provider.go

@@ -4,6 +4,9 @@
 package starlark
 
 import (
+	"context"
+	"fmt"
+
 	"github.com/pkg/errors"
 	"github.com/vmware-tanzu/crash-diagnostics/k8s"
 	"github.com/vmware-tanzu/crash-diagnostics/provider"
@@ -32,6 +35,11 @@ func CapaProviderFn(thread *starlark.Thread, _ *starlark.Builtin, args starlark.
 		return starlark.None, errors.Wrap(err, "failed to unpack input arguments")
 	}
 
+	ctx, ok := thread.Local(identifiers.scriptCtx).(context.Context)
+	if !ok || ctx == nil {
+		return starlark.None, fmt.Errorf("script context not found")
+	}
+
 	if sshConfig == nil || mgmtKubeConfig == nil {
 		return starlark.None, errors.New("capa_provider requires the name of the management cluster, the ssh configuration and the management cluster kubeconfig")
 	}
@@ -68,7 +76,7 @@ func CapaProviderFn(thread *starlark.Thread, _ *starlark.Builtin, args starlark.
 		return starlark.None, err
 	}
 
-	nodeAddresses, err := k8s.GetNodeAddresses(providerConfigPath, toSlice(names), toSlice(labels))
+	nodeAddresses, err := k8s.GetNodeAddresses(ctx, providerConfigPath, toSlice(names), toSlice(labels))
 	if err != nil {
 		return starlark.None, errors.Wrap(err, "could not fetch host addresses")
 	}

starlark/capv_provider.go

@@ -4,6 +4,9 @@
 package starlark
 
 import (
+	"context"
+	"fmt"
+
 	"github.com/pkg/errors"
 	"github.com/vmware-tanzu/crash-diagnostics/k8s"
 	"github.com/vmware-tanzu/crash-diagnostics/provider"
@@ -32,6 +35,11 @@ func CapvProviderFn(thread *starlark.Thread, _ *starlark.Builtin, args starlark.
 		return starlark.None, errors.Wrap(err, "failed to unpack input arguments")
 	}
 
+	ctx, ok := thread.Local(identifiers.scriptCtx).(context.Context)
+	if !ok || ctx == nil {
+		return starlark.None, fmt.Errorf("script context not found")
+	}
+
 	if sshConfig == nil || mgmtKubeConfig == nil {
 		return starlark.None, errors.New("capv_provider requires the name of the management cluster, the ssh configuration and the management cluster kubeconfig")
 	}
@@ -49,7 +57,7 @@ func CapvProviderFn(thread *starlark.Thread, _ *starlark.Builtin, args starlark.
 		return starlark.None, err
 	}
 
-	nodeAddresses, err := k8s.GetNodeAddresses(providerConfigPath, toSlice(names), toSlice(labels))
+	nodeAddresses, err := k8s.GetNodeAddresses(ctx, providerConfigPath, toSlice(names), toSlice(labels))
 	if err != nil {
 		return starlark.None, errors.Wrap(err, "could not fetch host addresses")
 	}