Multiple Example Starlark Scripts

This patch adds several example scripts - Host list provider - CAPI boot strap in kind cluster - Querying API objects - Kubernes nodes provider - Query logs via Api objects - Command to script argument passing The patch also introduced minor name changes. Signed-off-by: Vladimir Vivien <vivienv@vmware.com>
vmware-tanzu · Jul 16, 2020 · a2d6299 · a2d6299
1 parent 2bece74
commit a2d6299
Show file tree

Hide file tree

Showing 14 changed files with 293 additions and 43 deletions.
diff --git a/examples/host-list-provider.star b/examples/host-list-provider.star
@@ -0,0 +1,28 @@
+# Copyright (c) 2020 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# This script shows how to use the host list provider.
+# As its name implies, this provider takes a list of hosts
+# and allows command functions to execute on those hosts using
+# SSH.
+#
+# This example requires an SSH server running on the targeted hosts.
+
+# setup and configuration
+ssh=ssh_config(
+    username=os.username,
+    private_key_path="{0}/.ssh/id_rsa".format(os.home),
+    port=args.ssh_port,
+    max_retries=5,
+)
+
+provider=host_list_provider(hosts=["localhost", "127.0.0.1"], ssh_config=ssh)
+hosts=resources(provider=provider)
+
+# commands to run on each host
+uptimes = run(cmd="uptime", resources=hosts)
+
+# result for resource 0 (localhost)
+print(uptimes[0].result)
+# result for resource 1 (127.0.0.1)
+print(uptimes[1].result)
diff --git a/examples/kind-api-objects.star b/examples/kind-api-objects.star
@@ -0,0 +1,22 @@
+# Copyright (c) 2020 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+conf=crashd_config(workdir="/tmp/crashobjs")
+nspaces=[
+    "capi-kubeadm-bootstrap-system",
+    "capi-kubeadm-control-plane-system",
+    "capi-system capi-webhook-system",
+    "capv-system capa-system",
+    "cert-manager tkg-system",
+]
+
+
+kube_config(path=args.kubecfg)
+
+# capture Kubernetes API object and store in files (under working dir)
+kube_capture(what="objects", kinds=["services", "pods"], namespaces=nspaces)
+kube_capture(what="objects", kinds=["deployments", "replicasets"], namespaces=nspaces)
+kube_capture(what="objects", kinds=["clusters", "machines", "machinesets", "machinedeployments"], namespaces="tkg-system")
+
+# bundle files stored in working dir
+archive(output_file="/tmp/crashobjs.tar.gz", source_paths=[conf.workdir])
diff --git a/examples/kind-capi-bootstrap.star b/examples/kind-capi-bootstrap.star
@@ -0,0 +1,39 @@
+# Copyright (c) 2020 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Kind CAPI Bootstrap example
+# The following script extracts CAPI bootstrap info from a kind cluster.
+
+# declare global default config for script
+conf=crashd_config(workdir="/tmp/crashd-test")
+
+kind_cluster = args.cluster_name
+
+# exports kind logs to a file under workdir directory
+run_local("kind export logs --name {0} {1}/kind-logs".format(kind_cluster, conf.workdir))
+
+# runs `kind get kubeconfig` to capture kubeconfig file
+kind_cfg = capture_local(
+    cmd="kind get kubeconfig --name {0}".format(kind_cluster),
+    file_name="kind.kubecfg"
+)
+
+# declares default configuration for Kubernetes commands
+
+nspaces=[
+    "capi-kubeadm-bootstrap-system",
+    "capi-kubeadm-control-plane-system",
+    "capi-system capi-webhook-system",
+    "capv-system capa-system",
+    "cert-manager tkg-system",
+]
+
+kconf=kube_config(path=kind_cfg)
+
+# capture Kubernetes API object and store in files (under working dir)
+kube_capture(what="objects", kinds=["services", "pods"], namespaces=nspaces, kube_conf=kconf)
+kube_capture(what="objects", kinds=["deployments", "replicasets"], namespaces=nspaces, kube_conf=kconf)
+kube_capture(what="objects", kinds=["clusters", "machines", "machinesets", "machinedeployments"], namespaces="tkg-system", kube_conf=kconf)
+
+# bundle files stored in working dir
+archive(output_file="/tmp/crashout.tar.gz", source_paths=[conf.workdir])
diff --git a/examples/kube-nodes-provider.star b/examples/kube-nodes-provider.star
@@ -0,0 +1,30 @@
+# Copyright (c) 2020 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# This script shows how to use the kube nodes provider.
+# The kube node provider uses the Kubernetes Nodes objects
+# to enumerate compute resources that are part of the cluster.
+# It uses SSH to execute commands on those on nodes.
+#
+# This example requires an SSH and a Kubernetes cluster.
+
+# setup and configuration
+ssh=ssh_config(
+    username=os.username,
+    private_key_path="{0}/.ssh/id_rsa".format(os.home),
+    port=args.ssh_port,
+    max_retries=5,
+)
+
+hosts=resources(
+    provider=kube_nodes_provider(
+        kube_config=kube_config(path=args.kubecfg),
+        ssh_config=ssh,
+    ),
+)
+
+# commands to run on each host
+uptimes = run(cmd="uptime", resources=hosts)
+
+# result for resource 0 (localhost)
+print(uptimes.result)
diff --git a/examples/pod-logs.star b/examples/pod-logs.star
@@ -0,0 +1,9 @@
+# Copyright (c) 2020 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+conf=crashd_config(workdir="/tmp/crashlogs")
+kube_config(path="{0}/.kube/config".format(os.home))
+kube_capture(what="logs", namespaces=["default", "cert-manager", "tkg-system"])
+
+# bundle files stored in working dir
+archive(output_file="/tmp/craslogs.tar.gz", source_paths=[conf.workdir])
diff --git a/examples/script-args.star b/examples/script-args.star
@@ -0,0 +1,9 @@
+# Copyright (c) 2020 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+conf=crashd_config(workdir=args.workdir)
+kube_config(path=args.kubecfg)
+kube_capture(what="logs", namespaces=["default", "cert-manager", "tkg-system"])
+
+# bundle files stored in working dir
+archive(output_file=args.output, source_paths=[args.workdir])
diff --git a/exec/executor_test.go b/exec/executor_test.go
@@ -4,36 +4,150 @@
 package exec
 
 import (
+	"io/ioutil"
 	"os"
+	"strings"
 	"testing"
+	"time"
+
+	"github.com/sirupsen/logrus"
 
 	testcrashd "github.com/vmware-tanzu/crash-diagnostics/testing"
 )
 
-const (
-	testSSHPort = "2222"
+var (
+	testSSHPort     = testcrashd.NextPortValue()
+	testServerName  = testcrashd.NextResourceName()
+	testClusterName = testcrashd.NextResourceName()
+	getTestKubeConf func() string
 )
 
 func TestMain(m *testing.M) {
 	testcrashd.Init()
-	//
-	//sshSvr := testcrashd.NewSSHServer("test-sshd-exec", testSSHPort)
-	//logrus.Debug("Attempting to start SSH server")
-	//if err := sshSvr.Start(); err != nil {
-	//	logrus.Error(err)
-	//	os.Exit(1)
-	//}
-	//
-	//testResult := m.Run()
-	//
-	//logrus.Debug("Stopping SSH server...")
-	//if err := sshSvr.Stop(); err != nil {
-	//	logrus.Error(err)
-	//	os.Exit(1)
-	//}
-	//
-	//os.Exit(testResult)
-
-	// Skipping all tests
-	os.Exit(0)
+
+	sshSvr := testcrashd.NewSSHServer(testServerName, testSSHPort)
+	logrus.Debug("Attempting to start SSH server")
+	if err := sshSvr.Start(); err != nil {
+		logrus.Error(err)
+		os.Exit(1)
+	}
+
+	kind := testcrashd.NewKindCluster("../testing/kind-cluster-docker.yaml", testClusterName)
+	if err := kind.Create(); err != nil {
+		logrus.Error(err)
+		os.Exit(1)
+	}
+
+	// attempt to wait for cluster up
+	time.Sleep(time.Second * 10)
+
+	tmpFile, err := ioutil.TempFile(os.TempDir(), testClusterName)
+	if err != nil {
+		logrus.Error(err)
+		os.Exit(1)
+	}
+
+	defer func() {
+		logrus.Debug("Stopping SSH server...")
+		if err := sshSvr.Stop(); err != nil {
+			logrus.Error(err)
+			os.Exit(1)
+		}
+
+		if err := kind.Destroy(); err != nil {
+			logrus.Error(err)
+			os.Exit(1)
+		}
+	}()
+
+	getTestKubeConf = func() string {
+		return tmpFile.Name()
+	}
+
+	if err := kind.MakeKubeConfigFile(getTestKubeConf()); err != nil {
+		logrus.Error(err)
+		os.Exit(1)
+	}
+
+	os.Exit(m.Run())
+}
+
+func TestKindScript(t *testing.T) {
+	tests := []struct {
+		name       string
+		scriptPath string
+		args       ArgMap
+	}{
+		//{
+		//	name:       "api objects",
+		//	scriptPath: "../examples/kind-api-objects.star",
+		//	args:       ArgMap{"kubecfg": getTestKubeConf()},
+		//},
+		//{
+		//	name:       "pod logs",
+		//	scriptPath: "../examples/pod-logs.star",
+		//	args:       ArgMap{"kubecfg": getTestKubeConf()},
+		//},
+		//{
+		//	name:       "script with args",
+		//	scriptPath: "../examples/script-args.star",
+		//	args: ArgMap{
+		//		"workdir": "/tmp/crashargs",
+		//		"kubecfg": getTestKubeConf(),
+		//		"output":  "/tmp/craslogs.tar.gz",
+		//	},
+		//},
+		//{
+		//	name:       "host-list provider",
+		//	scriptPath: "../examples/host-list-provider.star",
+		//	args:       ArgMap{"kubecfg": getTestKubeConf(), "ssh_port": testSSHPort},
+		//},
+		{
+			name:       "kube-nodes provider",
+			scriptPath: "../examples/kube-nodes-provider.star",
+			args:       ArgMap{"kubecfg": getTestKubeConf(), "ssh_port": testSSHPort},
+		},
+		//{
+		//	name:       "kind-capi-bootstrap",
+		//	scriptPath: "../examples/kind-capi-bootstrap.star",
+		//	args:       ArgMap{"cluster_name": testClusterName},
+		//},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			file, err := os.Open(test.scriptPath)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer file.Close()
+			if err := ExecuteFile(file, test.args); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+func TestExecute(t *testing.T) {
+	tests := []struct {
+		name   string
+		script string
+		exec   func(t *testing.T, script string)
+	}{
+		{
+			name:   "run_local",
+			script: `result = run_local("echo 'Hello World!'")`,
+			exec: func(t *testing.T, script string) {
+				if err := Execute("run_local", strings.NewReader(script), ArgMap{}); err != nil {
+					t.Fatal(err)
+				}
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			test.exec(t, test.script)
+		})
+	}
 }
diff --git a/ssh/main_test.go b/ssh/main_test.go
@@ -13,14 +13,14 @@ import (
 )
 
 var (
-	testSSHPort    = testcrashd.NextSSHPort()
+	testSSHPort    = testcrashd.NextPortValue()
 	testMaxRetries = 30
 )
 
 func TestMain(m *testing.M) {
 	testcrashd.Init()
 
-	sshSvr := testcrashd.NewSSHServer(testcrashd.NextSSHContainerName(), testSSHPort)
+	sshSvr := testcrashd.NewSSHServer(testcrashd.NextResourceName(), testSSHPort)
 	logrus.Debug("Attempting to start SSH server")
 	if err := sshSvr.Start(); err != nil {
 		logrus.Error(err)

diff --git a/starlark/capture_test.go b/starlark/capture_test.go
@@ -274,8 +274,8 @@ result = exec(hosts)`, port),
 }
 
 func TestCaptureFuncSSHAll(t *testing.T) {
-	port := testcrashd.NextSSHPort()
-	sshSvr := testcrashd.NewSSHServer(testcrashd.NextSSHContainerName(), port)
+	port := testcrashd.NextPortValue()
+	sshSvr := testcrashd.NewSSHServer(testcrashd.NextResourceName(), port)
 
 	logrus.Debug("Attempting to start SSH server")
 	if err := sshSvr.Start(); err != nil {

diff --git a/starlark/copy_from_test.go b/starlark/copy_from_test.go
@@ -375,8 +375,8 @@ result = cp(hosts)`, port),
 }
 
 func TestCopyFuncSSHAll(t *testing.T) {
-	port := testcrashd.NextSSHPort()
-	sshSvr := testcrashd.NewSSHServer(testcrashd.NextSSHContainerName(), port)
+	port := testcrashd.NextPortValue()
+	sshSvr := testcrashd.NewSSHServer(testcrashd.NextResourceName(), port)
 
 	logrus.Debug("Attempting to start SSH server")
 	if err := sshSvr.Start(); err != nil {

diff --git a/starlark/os_builtins.go b/starlark/os_builtins.go
@@ -17,7 +17,7 @@ func setupOSStruct() *starlarkstruct.Struct {
 		starlark.StringDict{
 			"name":     starlark.String(runtime.GOOS),
 			"username": starlark.String(getUsername()),
-			"homedir":  starlark.String(os.Getenv("HOME")),
+			"home":     starlark.String(os.Getenv("HOME")),
 			"getenv":   starlark.NewBuiltin("getenv", getEnvFunc),
 		},
 	)

diff --git a/starlark/run_local.go b/starlark/run_local.go
@@ -15,18 +15,17 @@ import (
 // Starlark format: run_local(<command string>)
 func runLocalFunc(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
 	var cmdStr string
-	if args != nil && args.Len() == 1 {
-		cmd, ok := args.Index(0).(starlark.String)
-		if !ok {
-			return starlark.None, fmt.Errorf("%s: command must be a string", identifiers.runLocal)
-		}
-		cmdStr = string(cmd)
+	if err := starlark.UnpackArgs(
+		identifiers.runLocal, args, kwargs,
+		"cmd", &cmdStr,
+	); err != nil {
+		return starlark.None, fmt.Errorf("%s: %s", identifiers.run, err)
 	}
 
 	p := echo.New().RunProc(cmdStr)
 	if p.Err() != nil {
-		return starlark.None, fmt.Errorf("%s: %s", identifiers.runLocal, p.Err())
+		return starlark.None, fmt.Errorf("%s: %s: %s", identifiers.runLocal, p.Err(), p.Result())
 	}
 
 	return starlark.String(p.Result()), nil
-}
+}