From 7bfa1a0e9c23067557b6933abbaa7b2414849e4d Mon Sep 17 00:00:00 2001 From: Sagar Muchhal Date: Wed, 15 Jul 2020 01:04:15 -0700 Subject: [PATCH] New provider for CAPV resource enumeration This patch adds a new capv_provider() which is used to enumerate the compute resources for the management/or workload clusters managed by CAPV. It also extends the kube_config() directive to accept a CAPI provider implementation as an input, thus enabling workload cluster kube configs to be discoverable by the directive and used in the script subsequently. Also, this patch updates the generation of sshArgs by including the private key path. In addition, it also updates the SearchParams struct from the k8s package to accept the search parameters directly as golang slices. Adds private key path to ssh args --- cmd/run.go | 2 +- examples/capv_provider.file | 32 +++++++++ k8s/client.go | 12 +++- k8s/kube_config.go | 38 ++++++++++ k8s/nodes.go | 63 +++++++++++++++++ k8s/search_params.go | 93 ++++++++++--------------- k8s/search_params_test.go | 23 +++--- provider/kube_config.go | 26 +++++++ starlark/archive.go | 2 +- starlark/capv_provider.go | 86 +++++++++++++++++++++++ starlark/kube_capture.go | 35 ++-------- starlark/kube_config.go | 69 ++++++++++++++++-- starlark/kube_config_test.go | 119 ++++++++++++++++++++++---------- starlark/kube_get.go | 2 +- starlark/kube_nodes_provider.go | 52 ++------------ starlark/resources.go | 2 +- starlark/run.go | 15 ++-- starlark/starlark_exec.go | 3 +- starlark/support.go | 2 + 19 files changed, 477 insertions(+), 199 deletions(-) create mode 100644 examples/capv_provider.file create mode 100644 k8s/kube_config.go create mode 100644 k8s/nodes.go create mode 100644 provider/kube_config.go create mode 100644 starlark/capv_provider.go diff --git a/cmd/run.go b/cmd/run.go index 84b57ddc..58833cca 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -32,7 +32,7 @@ func newRunCommand() *cobra.Command { return run(flags) }, } - cmd.Flags().StringToStringVar(&flags.args, "args", flags.args, "space-separated key=value arguments to passed to diagnostics file") + cmd.Flags().StringToStringVar(&flags.args, "args", flags.args, "comma-separated key=value arguments to pass to the diagnostics file") cmd.Flags().StringVar(&flags.file, "file", flags.file, "the path to the diagnostics script file to run") return cmd } diff --git a/examples/capv_provider.file b/examples/capv_provider.file new file mode 100644 index 00000000..2f440d4a --- /dev/null +++ b/examples/capv_provider.file @@ -0,0 +1,32 @@ +conf = crashd_config(workdir=args.workdir) +ssh_conf = ssh_config(username="capv", private_key_path=args.private_key) +kube_conf = kube_config(path=args.mc_config) + +#list out management and workload cluster nodes +wc_provider=capv_provider( + workload_cluster=args.cluster_name, + ssh_config=ssh_conf, + kube_config=kube_conf +) +nodes = resources(provider=wc_provider) + +capture(cmd="sudo df -i", resources=nodes) +capture(cmd="sudo crictl info", resources=nodes) +capture(cmd="df -h /var/lib/containerd", resources=nodes) +capture(cmd="sudo systemctl status kubelet", resources=nodes) +capture(cmd="sudo systemctl status containerd", resources=nodes) +capture(cmd="sudo journalctl -xeu kubelet", resources=nodes) + +capture(cmd="sudo cat /var/log/cloud-init-output.log", resources=nodes) +capture(cmd="sudo cat /var/log/cloud-init.log", resources=nodes) + +#add code to collect pod info from cluster +wc_kube_conf = kube_config(capi_provider = wc_provider) + +pod_ns=["default", "kube-system"] + +kube_capture(what="logs", namespaces=pod_ns, kube_config=wc_kube_conf) +kube_capture(what="objects", kinds=["pods", "services"], namespaces=pod_ns, kube_config=wc_kube_conf) +kube_capture(what="objects", kinds=["deployments", "replicasets"], groups=["apps"], namespaces=pod_ns, kube_config=wc_kube_conf) + +archive(output_file="diagnostics.tar.gz", source_paths=[conf.workdir]) \ No newline at end of file diff --git a/k8s/client.go b/k8s/client.go index 5b9f2aa4..adb76f3a 100644 --- a/k8s/client.go +++ b/k8s/client.go @@ -66,13 +66,23 @@ func New(kubeconfig string) (*Client, error) { return &Client{Client: client, Disco: disco, CoreRest: restc}, nil } +func (k8sc *Client) Search(params SearchParams) ([]SearchResult, error) { + return k8sc._search(strings.Join(params.Groups, " "), + strings.Join(params.Kinds, " "), + strings.Join(params.Namespaces, " "), + strings.Join(params.Versions, " "), + strings.Join(params.Names, " "), + strings.Join(params.Labels, " "), + strings.Join(params.Containers, " ")) +} + // Search does a drill-down search from group, version, resourceList, to resources. The following rules are applied // 1) Legacy core group (api/v1) can be specified as "core" // 2) All specified search params will use AND operator for match (i.e. groups=core AND kinds=pods AND versions=v1 AND ... etc) // 3) kinds will match resource.Kind or resource.Name // 4) All search params are passed as comma- or space-separated sets that are matched using OR (i.e. kinds=pods services // will match resouces of type pods or services) -func (k8sc *Client) Search(groups, kinds, namespaces, versions, names, labels, containers string) ([]SearchResult, error) { +func (k8sc *Client) _search(groups, kinds, namespaces, versions, names, labels, containers string) ([]SearchResult, error) { // normalize params groups = strings.ToLower(groups) kinds = strings.ToLower(kinds) diff --git a/k8s/kube_config.go b/k8s/kube_config.go new file mode 100644 index 00000000..176d43a6 --- /dev/null +++ b/k8s/kube_config.go @@ -0,0 +1,38 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package k8s + +import ( + "encoding/base64" + "fmt" + "io" + "io/ioutil" + "os" + + "github.com/pkg/errors" + "github.com/vladimirvivien/echo" +) + +// FetchWorkloadConfig... +func FetchWorkloadConfig(name, mgmtKubeConfigPath string) (string, error) { + var filePath string + cmdStr := fmt.Sprintf(`kubectl get secrets/%s-kubeconfig --template '{{.data.value}}' --kubeconfig %s`, name, mgmtKubeConfigPath) + p := echo.New().RunProc(cmdStr) + if p.Err() != nil { + return filePath, fmt.Errorf("kubectl get secrets failed: %s: %s", p.Err(), p.Result()) + } + + f, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("%s-workload-config", name)) + if err != nil { + return filePath, errors.Wrap(err, "Cannot create temporary file") + } + filePath = f.Name() + defer f.Close() + + base64Dec := base64.NewDecoder(base64.StdEncoding, p.Out()) + if _, err := io.Copy(f, base64Dec); err != nil { + return filePath, errors.Wrap(err, "error decoding workload kubeconfig") + } + return filePath, nil +} diff --git a/k8s/nodes.go b/k8s/nodes.go new file mode 100644 index 00000000..0e63ec46 --- /dev/null +++ b/k8s/nodes.go @@ -0,0 +1,63 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package k8s + +import ( + "github.com/pkg/errors" + coreV1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func GetNodeAddresses(kubeconfigPath string, labels, names []string) ([]string, error) { + client, err := New(kubeconfigPath) + if err != nil { + return nil, errors.Wrap(err, "could not initialize search client") + } + + nodes, err := getNodes(client, names, labels) + if err != nil { + return nil, errors.Wrapf(err, "could not fetch nodes") + } + + var nodeIps []string + for _, node := range nodes { + nodeIps = append(nodeIps, getNodeInternalIP(node)) + } + return nodeIps, nil +} + +func getNodes(k8sc *Client, names, labels []string) ([]*coreV1.Node, error) { + nodeResults, err := k8sc.Search(SearchParams{ + Groups: []string{"core"}, + Kinds: []string{"nodes"}, + Names: names, + Labels: labels, + }) + if err != nil { + return nil, err + } + + // collate + var nodes []*coreV1.Node + for _, result := range nodeResults { + for _, item := range result.List.Items { + node := new(coreV1.Node) + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(item.Object, &node); err != nil { + return nil, err + } + nodes = append(nodes, node) + } + } + return nodes, nil +} + +func getNodeInternalIP(node *coreV1.Node) (ipAddr string) { + for _, addr := range node.Status.Addresses { + if addr.Type == "InternalIP" { + ipAddr = addr.Address + return + } + } + return +} diff --git a/k8s/search_params.go b/k8s/search_params.go index d4807f6a..c74f91a3 100644 --- a/k8s/search_params.go +++ b/k8s/search_params.go @@ -10,69 +10,46 @@ import ( ) type SearchParams struct { - groups []string - kinds []string - namespaces []string - versions []string - names []string - labels []string - containers []string + Groups []string + Kinds []string + Namespaces []string + Versions []string + Names []string + Labels []string + Containers []string } -func (sp SearchParams) SetGroups(input []string) { - sp.groups = input +func (sp SearchParams) ContainsGroup(group string) bool { + return contains(sp.Groups, group) } -func (sp SearchParams) SetKinds(input []string) { - sp.kinds = input +func (sp SearchParams) ContainsVersion(version string) bool { + return contains(sp.Versions, version) } -func (sp SearchParams) SetNames(input []string) { - sp.names = input +func (sp SearchParams) ContainsKind(kind string) bool { + return contains(sp.Kinds, kind) } -func (sp SearchParams) SetNamespaces(input []string) { - sp.namespaces = input +func (sp SearchParams) ContainsContainer(container string) bool { + return contains(sp.Containers, container) } -func (sp SearchParams) SetVersions(input []string) { - sp.versions = input +func (sp SearchParams) ContainsName(name string) bool { + return contains(sp.Names, name) } -func (sp SearchParams) SetLabels(input []string) { - sp.labels = input -} - -func (sp SearchParams) SetContainers(input []string) { - sp.containers = input -} - -func (sp SearchParams) Groups() string { - return strings.Join(sp.groups, " ") -} - -func (sp SearchParams) Kinds() string { - return strings.Join(sp.kinds, " ") -} - -func (sp SearchParams) Names() string { - return strings.Join(sp.names, " ") -} - -func (sp SearchParams) Namespaces() string { - return strings.Join(sp.namespaces, " ") -} - -func (sp SearchParams) Versions() string { - return strings.Join(sp.versions, " ") -} - -func (sp SearchParams) Labels() string { - return strings.Join(sp.labels, " ") -} - -func (sp SearchParams) Containers() string { - return strings.Join(sp.containers, " ") +// contains performs a case-insensitive search for the item in the input array +func contains(arr []string, item string) bool { + if len(arr) == 0 { + return false + } + for _, str := range arr { + if strings.ToLower(str) == strings.ToLower(item) { + return true + } + } + return false } // TODO: Change this to accept a string dictionary instead @@ -99,13 +76,13 @@ func NewSearchParams(p *starlarkstruct.Struct) SearchParams { containers = parseStructAttr(p, "containers") return SearchParams{ - kinds: kinds, - groups: groups, - names: names, - namespaces: namespaces, - versions: versions, - labels: labels, - containers: containers, + Kinds: kinds, + Groups: groups, + Names: names, + Namespaces: namespaces, + Versions: versions, + Labels: labels, + Containers: containers, } } diff --git a/k8s/search_params_test.go b/k8s/search_params_test.go index b6be1b26..6cc65674 100644 --- a/k8s/search_params_test.go +++ b/k8s/search_params_test.go @@ -36,8 +36,8 @@ var _ = Describe("SearchParams", func() { input = starlarkstruct.FromStringDict(starlarkstruct.Default, args) searchParams = NewSearchParams(input) Expect(searchParams).To(BeAssignableToTypeOf(SearchParams{})) - Expect(searchParams.kinds).To(HaveLen(1)) - Expect(searchParams.Kinds()).To(Equal("deployments")) + Expect(searchParams.Kinds).To(HaveLen(1)) + Expect(searchParams.Kinds).To(ConsistOf("deployments")) }) It("returns a new instance with kinds struct member populated", func() { @@ -47,8 +47,8 @@ var _ = Describe("SearchParams", func() { input = starlarkstruct.FromStringDict(starlarkstruct.Default, args) searchParams = NewSearchParams(input) Expect(searchParams).To(BeAssignableToTypeOf(SearchParams{})) - Expect(searchParams.kinds).To(HaveLen(2)) - Expect(searchParams.Kinds()).To(Equal("deployments replicasets")) + Expect(searchParams.Kinds).To(HaveLen(2)) + Expect(searchParams.Kinds).To(ConsistOf("deployments", "replicasets")) }) }) @@ -58,8 +58,7 @@ var _ = Describe("SearchParams", func() { input = starlarkstruct.FromStringDict(starlarkstruct.Default, starlark.StringDict{}) searchParams = NewSearchParams(input) Expect(searchParams).To(BeAssignableToTypeOf(SearchParams{})) - Expect(searchParams.kinds).To(HaveLen(0)) - Expect(searchParams.Kinds()).To(Equal("")) + Expect(searchParams.Kinds).To(HaveLen(0)) }) }) }) @@ -75,8 +74,8 @@ var _ = Describe("SearchParams", func() { input = starlarkstruct.FromStringDict(starlarkstruct.Default, args) searchParams = NewSearchParams(input) Expect(searchParams).To(BeAssignableToTypeOf(SearchParams{})) - Expect(searchParams.namespaces).To(HaveLen(1)) - Expect(searchParams.Namespaces()).To(Equal("foo")) + Expect(searchParams.Namespaces).To(HaveLen(1)) + Expect(searchParams.Namespaces).To(ConsistOf("foo")) }) It("returns a new instance with namespaces struct member populated", func() { @@ -86,8 +85,8 @@ var _ = Describe("SearchParams", func() { input = starlarkstruct.FromStringDict(starlarkstruct.Default, args) searchParams = NewSearchParams(input) Expect(searchParams).To(BeAssignableToTypeOf(SearchParams{})) - Expect(searchParams.namespaces).To(HaveLen(2)) - Expect(searchParams.Namespaces()).To(Equal("foo bar")) + Expect(searchParams.Namespaces).To(HaveLen(2)) + Expect(searchParams.Namespaces).To(ConsistOf("foo", "bar")) }) }) @@ -97,8 +96,8 @@ var _ = Describe("SearchParams", func() { input = starlarkstruct.FromStringDict(starlarkstruct.Default, starlark.StringDict{}) searchParams = NewSearchParams(input) Expect(searchParams).To(BeAssignableToTypeOf(SearchParams{})) - Expect(searchParams.namespaces).To(HaveLen(1)) - Expect(searchParams.Namespaces()).To(Equal("default")) + Expect(searchParams.Namespaces).To(HaveLen(1)) + Expect(searchParams.Namespaces).To(ConsistOf("default")) }) }) }) diff --git a/provider/kube_config.go b/provider/kube_config.go new file mode 100644 index 00000000..9778cbd8 --- /dev/null +++ b/provider/kube_config.go @@ -0,0 +1,26 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package provider + +import ( + "fmt" + + "github.com/pkg/errors" + "github.com/vmware-tanzu/crash-diagnostics/k8s" +) + +// KubeConfig returns the kubeconfig that needs to be used by the provider. +// The path of the management kubeconfig file gets returned if the workload cluster name is empty +func KubeConfig(mgmtKubeConfigPath, workloadClusterName string) (string, error) { + var err error + + kubeConfigPath := mgmtKubeConfigPath + if len(workloadClusterName) != 0 { + kubeConfigPath, err = k8s.FetchWorkloadConfig(workloadClusterName, mgmtKubeConfigPath) + if err != nil { + err = errors.Wrap(err, fmt.Sprintf("could not fetch kubeconfig for workload cluster %s", workloadClusterName)) + } + } + return kubeConfigPath, err +} diff --git a/starlark/archive.go b/starlark/archive.go index ac1d3ff1..ea28d22b 100644 --- a/starlark/archive.go +++ b/starlark/archive.go @@ -13,7 +13,7 @@ import ( // archiveFunc is a built-in starlark function that bundles specified directories into // an arhive format (i.e. tar.gz) -// Starlark format: archive(file_name= ,source_paths=list) +// Starlark format: archive(output_file= ,source_paths=list) func archiveFunc(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var outputFile string var paths *starlark.List diff --git a/starlark/capv_provider.go b/starlark/capv_provider.go new file mode 100644 index 00000000..d5fc59c5 --- /dev/null +++ b/starlark/capv_provider.go @@ -0,0 +1,86 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package starlark + +import ( + "github.com/pkg/errors" + "github.com/vmware-tanzu/crash-diagnostics/k8s" + "github.com/vmware-tanzu/crash-diagnostics/provider" + "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" +) + +// CapvProviderFn is a built-in starlark function that collects compute resources from a k8s cluster +// Starlark format: capv_provider(kube_config=kube_config(), ssh_config=ssh_config()[workload_cluster=, nodes=["foo", "bar], labels=["bar", "baz"]]) +func CapvProviderFn(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + + var ( + workloadCluster string + names, labels *starlark.List + sshConfig, kubeConfig *starlarkstruct.Struct + ) + + err := starlark.UnpackArgs("capv_provider", args, kwargs, + "ssh_config", &sshConfig, + "kube_config", &kubeConfig, + "workload_cluster?", &workloadCluster, + "labels?", &labels, + "nodes?", &names) + if err != nil { + return starlark.None, errors.Wrap(err, "failed to unpack input arguments") + } + + if sshConfig == nil || kubeConfig == nil { + return starlark.None, errors.New("capv_provider requires the name of the management cluster, the ssh configuration and the management cluster kubeconfig") + } + + mgmtKubeConfigPath, err := getKubeConfigFromStruct(kubeConfig) + if err != nil { + return starlark.None, errors.Wrap(err, "failed to extract management kubeconfig") + } + + providerConfigPath, err := provider.KubeConfig(mgmtKubeConfigPath, workloadCluster) + if err != nil { + return starlark.None, err + } + + nodeAddresses, err := k8s.GetNodeAddresses(providerConfigPath, toSlice(names), toSlice(labels)) + if err != nil { + return starlark.None, errors.Wrap(err, "could not fetch host addresses") + } + + // dictionary for capv provider struct + capvProviderDict := starlark.StringDict{ + "kind": starlark.String(identifiers.capvProvider), + "transport": starlark.String("ssh"), + "kubeconfig": starlark.String(providerConfigPath), + } + + // add node info to dictionary + var nodeIps []starlark.Value + for _, node := range nodeAddresses { + nodeIps = append(nodeIps, starlark.String(node)) + } + capvProviderDict["hosts"] = starlark.NewList(nodeIps) + + // add ssh info to dictionary + if _, ok := capvProviderDict[identifiers.sshCfg]; !ok { + capvProviderDict[identifiers.sshCfg] = sshConfig + } + + return starlarkstruct.FromStringDict(starlark.String(identifiers.capvProvider), capvProviderDict), nil +} + +// TODO: Needs to be moved to a single package +func toSlice(list *starlark.List) []string { + var elems []string + if list != nil { + for i := 0; i < list.Len(); i++ { + if val, ok := list.Index(i).(starlark.String); ok { + elems = append(elems, string(val)) + } + } + } + return elems +} diff --git a/starlark/kube_capture.go b/starlark/kube_capture.go index 20d4d243..5c571287 100644 --- a/starlark/kube_capture.go +++ b/starlark/kube_capture.go @@ -68,15 +68,15 @@ func write(workdir string, client *k8s.Client, structVal *starlarkstruct.Struct) logrus.Debugf("kube_capture(what=%s)", what) switch what { case "logs": - searchParams.SetGroups([]string{"core"}) - searchParams.SetKinds([]string{"pods"}) - searchParams.SetVersions([]string{}) + searchParams.Groups = []string{"core"} + searchParams.Kinds = []string{"pods"} + searchParams.Versions = []string{} case "objects", "all", "*": default: return "", errors.Errorf("don't know how to get: %s", what) } - searchResults, err = client.Search(searchParams.Groups(), searchParams.Kinds(), searchParams.Namespaces(), searchParams.Versions(), searchParams.Names(), searchParams.Labels(), searchParams.Containers()) + searchResults, err = client.Search(searchParams) if err != nil { return "", err } @@ -91,30 +91,3 @@ func write(workdir string, client *k8s.Client, structVal *starlarkstruct.Struct) } return resultWriter.GetResultDir(), nil } - -// getKubeConfigPath is responsible to obtain the path to the kubeconfig -// It checks for the `path` key in the input args for the directive otherwise -// falls back to the default kube_config from the thread context -func getKubeConfigPath(thread *starlark.Thread, structVal *starlarkstruct.Struct) (string, error) { - var ( - kubeConfigPath string - err error - kcVal starlark.Value - ) - - if kcVal, err = structVal.Attr("kube_config"); err != nil { - kubeConfigData := thread.Local(identifiers.kubeCfg) - kcVal = kubeConfigData.(starlark.Value) - } - - if kubeConfigVal, ok := kcVal.(*starlarkstruct.Struct); ok { - kvPathVal, err := kubeConfigVal.Attr("path") - if err != nil { - return kubeConfigPath, errors.Wrap(err, "unable to extract kubeconfig path") - } - if kvPathStrVal, ok := kvPathVal.(starlark.String); ok { - kubeConfigPath = kvPathStrVal.GoString() - } - } - return trimQuotes(kubeConfigPath), nil -} diff --git a/starlark/kube_config.go b/starlark/kube_config.go index ccd59b57..e2d157e5 100644 --- a/starlark/kube_config.go +++ b/starlark/kube_config.go @@ -6,22 +6,50 @@ package starlark import ( "fmt" + "github.com/pkg/errors" "go.starlark.net/starlark" "go.starlark.net/starlarkstruct" ) -// kubeConfigFn is built-in starlark function that wraps the kwargs into a dictionary value. +// KubeConfigFn is built-in starlark function that wraps the kwargs into a dictionary value. // The result is also added to the thread for other built-in to access. // Starlark: kube_config(path=kubecf/path) -func kubeConfigFn(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { +func KubeConfigFn(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var path string + var provider *starlarkstruct.Struct + if err := starlark.UnpackArgs( identifiers.crashdCfg, args, kwargs, - "path", &path, + "path?", &path, + "capi_provider?", &provider, ); err != nil { return starlark.None, fmt.Errorf("%s: %s", identifiers.kubeCfg, err) } + // check if only one of the two options are present + if (len(path) == 0 && provider == nil) || (len(path) != 0 && provider != nil) { + return starlark.None, errors.New("need either path or capi_provider") + } + + if len(path) == 0 { + val := provider.Constructor() + if constructor, ok := val.(starlark.String); ok { + if constructor.GoString() != identifiers.capvProvider { + return starlark.None, errors.New("unknown capi provider") + } + } + + pathVal, err := provider.Attr("kubeconfig") + if err != nil { + return starlark.None, errors.Wrap(err, "could not find the kubeconfig attribute") + } + pathStr, ok := pathVal.(starlark.String) + if !ok { + return starlark.None, errors.New("could not fetch kubeconfig") + } + path = pathStr.GoString() + } + structVal := starlarkstruct.FromStringDict(starlarkstruct.Default, starlark.StringDict{ "path": starlark.String(path), }) @@ -39,10 +67,43 @@ func addDefaultKubeConf(thread *starlark.Thread) error { {starlark.String("path"), starlark.String(defaults.kubeconfig)}, } - _, err := kubeConfigFn(thread, nil, nil, args) + _, err := KubeConfigFn(thread, nil, nil, args) if err != nil { return err } return nil } + +// getKubeConfigPath is responsible to obtain the path to the kubeconfig +// It checks for the `path` key in the input args for the directive otherwise +// falls back to the default kube_config from the thread context +func getKubeConfigPath(thread *starlark.Thread, structVal *starlarkstruct.Struct) (string, error) { + var ( + err error + kcVal starlark.Value + ) + + if kcVal, err = structVal.Attr("kube_config"); err != nil { + kubeConfigData := thread.Local(identifiers.kubeCfg) + kcVal = kubeConfigData.(starlark.Value) + } + + kubeConfigVal, ok := kcVal.(*starlarkstruct.Struct) + if !ok { + return "", err + } + return getKubeConfigFromStruct(kubeConfigVal) +} + +func getKubeConfigFromStruct(kubeConfigStructVal *starlarkstruct.Struct) (string, error) { + kvPathVal, err := kubeConfigStructVal.Attr("path") + if err != nil { + return "", errors.Wrap(err, "failed to extract kubeconfig path") + } + kvPathStrVal, ok := kvPathVal.(starlark.String) + if !ok { + return "", errors.New("failed to extract management kubeconfig") + } + return kvPathStrVal.GoString(), nil +} diff --git a/starlark/kube_config_test.go b/starlark/kube_config_test.go index 23aefbd3..38d94d98 100644 --- a/starlark/kube_config_test.go +++ b/starlark/kube_config_test.go @@ -6,10 +6,10 @@ package starlark import ( "strings" - "go.starlark.net/starlarkstruct" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" ) var _ = Describe("kube_config", func() { @@ -26,52 +26,59 @@ var _ = Describe("kube_config", func() { Expect(err).To(BeNil()) } - Context("With kube_config set in the script", func() { + It("throws an error when empty kube_config is used", func() { + err = New().Exec("test.kube.config", strings.NewReader(`kube_config()`)) + Expect(err).To(HaveOccurred()) + }) - BeforeEach(func() { - crashdScript = `kube_config(path="/foo/bar/kube/config")` - execSetup() - }) + Context("With path", func() { + Context("With kube_config set in the script", func() { - It("sets the kube_config in the starlark thread", func() { - kubeConfigData := executor.thread.Local(identifiers.kubeCfg) - Expect(kubeConfigData).NotTo(BeNil()) - }) + BeforeEach(func() { + crashdScript = `kube_config(path="/foo/bar/kube/config")` + execSetup() + }) - It("sets the path to the kubeconfig file", func() { - kubeConfigData := executor.thread.Local(identifiers.kubeCfg) - Expect(kubeConfigData).To(BeAssignableToTypeOf(&starlarkstruct.Struct{})) + It("sets the kube_config in the starlark thread", func() { + kubeConfigData := executor.thread.Local(identifiers.kubeCfg) + Expect(kubeConfigData).NotTo(BeNil()) + }) - cfg, _ := kubeConfigData.(*starlarkstruct.Struct) - Expect(cfg.AttrNames()).To(HaveLen(1)) + It("sets the path to the kubeconfig file", func() { + kubeConfigData := executor.thread.Local(identifiers.kubeCfg) + Expect(kubeConfigData).To(BeAssignableToTypeOf(&starlarkstruct.Struct{})) - val, err := cfg.Attr("path") - Expect(err).To(BeNil()) - Expect(trimQuotes(val.String())).To(Equal("/foo/bar/kube/config")) + cfg, _ := kubeConfigData.(*starlarkstruct.Struct) + Expect(cfg.AttrNames()).To(HaveLen(1)) + + val, err := cfg.Attr("path") + Expect(err).To(BeNil()) + Expect(trimQuotes(val.String())).To(Equal("/foo/bar/kube/config")) + }) }) - }) - Context("With kube_config returned as a value", func() { + Context("With kube_config returned as a value", func() { - BeforeEach(func() { - crashdScript = `cfg = kube_config(path="/foo/bar/kube/config")` - execSetup() - }) + BeforeEach(func() { + crashdScript = `cfg = kube_config(path="/foo/bar/kube/config")` + execSetup() + }) - It("returns the kube config as a result", func() { - Expect(executor.result.Has("cfg")).NotTo(BeNil()) - }) + It("returns the kube config as a result", func() { + Expect(executor.result.Has("cfg")).NotTo(BeNil()) + }) - It("also sets the kube_config in the starlark thread", func() { - kubeConfigData := executor.thread.Local(identifiers.kubeCfg) - Expect(kubeConfigData).NotTo(BeNil()) + It("also sets the kube_config in the starlark thread", func() { + kubeConfigData := executor.thread.Local(identifiers.kubeCfg) + Expect(kubeConfigData).NotTo(BeNil()) - cfg, _ := kubeConfigData.(*starlarkstruct.Struct) - Expect(cfg.AttrNames()).To(HaveLen(1)) + cfg, _ := kubeConfigData.(*starlarkstruct.Struct) + Expect(cfg.AttrNames()).To(HaveLen(1)) - val, err := cfg.Attr("path") - Expect(err).To(BeNil()) - Expect(trimQuotes(val.String())).To(Equal("/foo/bar/kube/config")) + val, err := cfg.Attr("path") + Expect(err).To(BeNil()) + Expect(trimQuotes(val.String())).To(Equal("/foo/bar/kube/config")) + }) }) }) @@ -95,3 +102,43 @@ var _ = Describe("kube_config", func() { }) }) }) + +var _ = Describe("KubeConfigFn", func() { + + Context("With capi_provider", func() { + + It("populates the path from the capi provider", func() { + val, err := KubeConfigFn(&starlark.Thread{Name: "test.kube.config.fn"}, nil, nil, + []starlark.Tuple{ + []starlark.Value{ + starlark.String("capi_provider"), + starlarkstruct.FromStringDict(starlark.String(identifiers.capvProvider), starlark.StringDict{ + "kubeconfig": starlark.String("/foo/bar"), + }), + }, + }) + Expect(err).NotTo(HaveOccurred()) + + cfg, _ := val.(*starlarkstruct.Struct) + Expect(cfg.AttrNames()).To(HaveLen(1)) + + path, err := cfg.Attr("path") + Expect(err).To(BeNil()) + Expect(trimQuotes(path.String())).To(Equal("/foo/bar")) + }) + + It("throws an error when an unknown provider is passed", func() { + _, err := KubeConfigFn(&starlark.Thread{Name: "test.kube.config.fn"}, nil, nil, + []starlark.Tuple{ + []starlark.Value{ + starlark.String("capi_provider"), + starlarkstruct.FromStringDict(starlark.String("meh"), starlark.StringDict{ + "kubeconfig": starlark.String("/foo/bar"), + }), + }, + }) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError("unknown capi provider")) + }) + }) +}) diff --git a/starlark/kube_get.go b/starlark/kube_get.go index d3af8b9c..adb764c4 100644 --- a/starlark/kube_get.go +++ b/starlark/kube_get.go @@ -29,7 +29,7 @@ func KubeGetFn(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple } searchParams := k8s.NewSearchParams(structVal) - searchResults, err := client.Search(searchParams.Groups(), searchParams.Kinds(), searchParams.Namespaces(), searchParams.Versions(), searchParams.Names(), searchParams.Labels(), searchParams.Containers()) + searchResults, err := client.Search(searchParams) if err == nil { objects = starlark.NewList([]starlark.Value{}) for _, searchResult := range searchResults { diff --git a/starlark/kube_nodes_provider.go b/starlark/kube_nodes_provider.go index 2ddf3566..5130280e 100644 --- a/starlark/kube_nodes_provider.go +++ b/starlark/kube_nodes_provider.go @@ -8,8 +8,6 @@ import ( "github.com/pkg/errors" "github.com/vmware-tanzu/crash-diagnostics/k8s" - coreV1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" "go.starlark.net/starlark" "go.starlark.net/starlarkstruct" @@ -33,15 +31,11 @@ func newKubeNodesProvider(thread *starlark.Thread, structVal *starlarkstruct.Str if err != nil { return nil, errors.Wrap(err, "failed to kubeconfig") } - client, err := k8s.New(kubeconfig) - if err != nil { - return nil, errors.Wrap(err, "could not initialize search client") - } searchParams := generateSearchParams(structVal) - nodes, err := getNodes(client, searchParams.Names(), searchParams.Labels()) + nodeAddresses, err := k8s.GetNodeAddresses(kubeconfig, searchParams.Names, searchParams.Labels) if err != nil { - return nil, errors.Wrapf(err, "could not fetch nodes") + return nil, errors.Wrapf(err, "could not fetch node addresses") } // dictionary for node provider struct @@ -52,8 +46,8 @@ func newKubeNodesProvider(thread *starlark.Thread, structVal *starlarkstruct.Str // add node info to dictionary var nodeIps []starlark.Value - for _, node := range nodes { - nodeIps = append(nodeIps, starlark.String(getNodeInternalIP(node))) + for _, node := range nodeAddresses { + nodeIps = append(nodeIps, starlark.String(node)) } kubeNodesProviderDict["hosts"] = starlark.NewList(nodeIps) @@ -81,41 +75,3 @@ func generateSearchParams(structVal *starlarkstruct.Struct) k8s.SearchParams { } return k8s.NewSearchParams(structVal) } - -func getNodes(k8sc *k8s.Client, names, labels string) ([]*coreV1.Node, error) { - nodeResults, err := k8sc.Search( - "core", // group - "nodes", // kind - "", // namespaces - "", // version - names, - labels, - "", // containers - ) - if err != nil { - return nil, err - } - - // collate - var nodes []*coreV1.Node - for _, result := range nodeResults { - for _, item := range result.List.Items { - node := new(coreV1.Node) - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(item.Object, &node); err != nil { - return nil, err - } - nodes = append(nodes, node) - } - } - return nodes, nil -} - -func getNodeInternalIP(node *coreV1.Node) (ipAddr string) { - for _, addr := range node.Status.Addresses { - if addr.Type == "InternalIP" { - ipAddr = addr.Address - return - } - } - return -} diff --git a/starlark/resources.go b/starlark/resources.go index a3ee81ee..b6afcdb0 100644 --- a/starlark/resources.go +++ b/starlark/resources.go @@ -68,7 +68,7 @@ func enum(provider *starlarkstruct.Struct) (*starlark.List, error) { kind := trimQuotes(kindVal.String()) switch kind { - case identifiers.hostListProvider, identifiers.kubeNodesProvider: + case identifiers.hostListProvider, identifiers.kubeNodesProvider, identifiers.capvProvider: hosts, err := provider.Attr("hosts") if err != nil { return nil, fmt.Errorf("hosts not found in %s", identifiers.hostListProvider) diff --git a/starlark/run.go b/starlark/run.go index 65960831..da277bb5 100644 --- a/starlark/run.go +++ b/starlark/run.go @@ -194,11 +194,18 @@ func getSSHArgsFromCfg(sshCfg *starlarkstruct.Struct) (ssh.SSHArgs, error) { } } + var privateKeyPath string + if pkPathVal, err := sshCfg.Attr("private_key_path"); err == nil { + pkPath := pkPathVal.(starlark.String) + privateKeyPath = pkPath.GoString() + } + args := ssh.SSHArgs{ - User: string(user), - Port: port, - MaxRetries: maxRetries, - ProxyJump: jumpProxy, + User: string(user), + Port: port, + MaxRetries: maxRetries, + ProxyJump: jumpProxy, + PrivateKeyPath: privateKeyPath, } return args, nil } diff --git a/starlark/starlark_exec.go b/starlark/starlark_exec.go index c9004125..b989f597 100644 --- a/starlark/starlark_exec.go +++ b/starlark/starlark_exec.go @@ -87,10 +87,11 @@ func newPredeclareds() starlark.StringDict { identifiers.capture: starlark.NewBuiltin(identifiers.capture, captureFunc), identifiers.captureLocal: starlark.NewBuiltin(identifiers.capture, captureLocalFunc), identifiers.copyFrom: starlark.NewBuiltin(identifiers.copyFrom, copyFromFunc), - identifiers.kubeCfg: starlark.NewBuiltin(identifiers.kubeCfg, kubeConfigFn), + identifiers.kubeCfg: starlark.NewBuiltin(identifiers.kubeCfg, KubeConfigFn), identifiers.kubeCapture: starlark.NewBuiltin(identifiers.kubeGet, KubeCaptureFn), identifiers.kubeGet: starlark.NewBuiltin(identifiers.kubeGet, KubeGetFn), identifiers.kubeNodesProvider: starlark.NewBuiltin(identifiers.kubeNodesProvider, KubeNodesProviderFn), + identifiers.capvProvider: starlark.NewBuiltin(identifiers.capvProvider, CapvProviderFn), } } diff --git a/starlark/support.go b/starlark/support.go index c5f390fc..4bb2cbb9 100644 --- a/starlark/support.go +++ b/starlark/support.go @@ -43,6 +43,7 @@ var ( kubeCapture string kubeGet string kubeNodesProvider string + capvProvider string }{ crashdCfg: "crashd_config", kubeCfg: "kube_config", @@ -68,6 +69,7 @@ var ( kubeCapture: "kube_capture", kubeGet: "kube_get", kubeNodesProvider: "kube_nodes_provider", + capvProvider: "capv_provider", } defaults = struct {