Merge pull request #141 from ravisantoshgudimetla/save-log-files

[ci] Save log files from output run

internal/test/framework/framework.go

@@ -3,8 +3,10 @@ package framework
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -28,6 +30,9 @@ const (
 
 	// awsUsername is the default windows username on AWS
 	awsUsername = "Administrator"
+	// remoteLogPath is the directory where all the log files related to components that we need are generated on the
+	// Windows VM
+	remoteLogPath = "C:\\k\\log\\"
 )
 
 var (
@@ -294,6 +299,76 @@ func (f *TestFramework) GetNode(externalIP string) (*v1.Node, error) {
 	return matchedNode, nil
 }
 
+// WriteToArtifactDir will write contents to $ARTIFACT_DIR/subDirName/filename. If subDirName is empty, contents
+// will be written to $ARTIFACT_DIR/filename
+func (f *TestFramework) WriteToArtifactDir(contents []byte, subDirName, filename string) error {
+	path := filepath.Join(artifactDir, subDirName, filename)
+	dir, _ := filepath.Split(path)
+	err := os.MkdirAll(dir, os.ModePerm)
+	if err != nil {
+		return fmt.Errorf("could not create %s: %s", dir, err)
+	}
+	return ioutil.WriteFile(path, contents, os.ModePerm)
+}
+
+// GetNode uses external IP and finds out the name associated with the node
+func (f *TestFramework) GetNodeName(externalIP string) (string, error) {
+	node, err := f.GetNode(externalIP)
+	if err != nil {
+		return "", fmt.Errorf("error while getting required kubernetes node object: %v", err)
+	}
+	return node.Name, nil
+}
+
+// RetrieveArtifacts should retrieve artifacts related the test run. Ideally this should retrieve all the logs related
+// to the Windows VM. This shouldn't return an error but should print the failures as log collection is nice to have
+// rather than a must have.
+// TODO: Think about how we can retrieve stdout from ansible out within this function
+func (f *TestFramework) RetrieveArtifacts() {
+	for i, vm := range f.WinVMs {
+		if vm == nil {
+			continue
+		}
+		if vm.GetCredentials() == nil {
+			log.Printf("no credentials provided for vm %d ", i)
+			continue
+		}
+
+		instanceID := vm.GetCredentials().GetInstanceId()
+		if len(instanceID) == 0 {
+			log.Printf("no instance id provided for vm %d", i)
+			continue
+		}
+
+		externalIP := vm.GetCredentials().GetIPAddress()
+		if len(externalIP) == 0 {
+			log.Printf("no external ip address found for the vm with instance ID %s", instanceID)
+			continue
+		}
+
+		nodeName, err := f.GetNodeName(externalIP)
+		if err != nil {
+			log.Printf("error while getting node name associated with the vm %s: %v", instanceID, err)
+		}
+
+		// We want a format like "nodes/ip-10-0-141-99.ec2.internal/logs/wsu/kubelet"
+		localKubeletLogPath := filepath.Join(artifactDir, "nodes", nodeName, "logs")
+
+		// Let's reinitialize the ssh client as hybrid overlay is known to cause ssh connections to be dropped
+		// TODO: Reduce the usage of Reinitialize as much as possible, this is to ensure that when we move to operator
+		// 		model, the reconnectivity should be handled automatically.
+		if err := vm.Reinitialize(); err != nil {
+			log.Printf("failed re-initializing ssh connectivity with on vm %s: %v", instanceID, err)
+		}
+		// Get the VM's private ip and populate log files in the test container.
+		// Make this a map["'"artifact_that_we_want_to_pull"]="log_file.name"
+		if err := vm.RetrieveFiles(remoteLogPath, localKubeletLogPath); err != nil {
+			log.Printf("failed retrieving log files on vm %s: %v", instanceID, err)
+			continue
+		}
+	}
+}
+
 // TearDown destroys the resources created by the Setup function
 func (f *TestFramework) TearDown() {
 	if f.noTeardown || f.WinVMs == nil {

internal/test/framework/windowsvm.go

@@ -46,6 +46,9 @@ type WindowsVM interface {
 	// CopyFile copies the given file to the remote directory in the Windows VM. The remote directory is created if it
 	// does not exist
 	CopyFile(string, string) error
+	// RetrieveFiles retrieves the list of file from the directory in the remote Windows VM to the local host. As of
+	// now, we're limiting every file in the remote directory to be written to single directory on the local host
+	RetrieveFiles(string, string) error
 	// Run executes the given command remotely on the Windows VM and returns the output of stdout and stderr. If the
 	// bool is set, it implies that the cmd is to be execute in PowerShell.
 	Run(string, bool) (string, string, error)
@@ -146,6 +149,74 @@ func (w *windowsVM) CopyFile(filePath, remoteDir string) error {
 	return nil
 }
 
+// RetrieveFiles retrieves list of files from remote directory to the local directory.
+// The implementation can be changed if the use-case arises. As of now, we're doing a best effort
+// to collect every log possible. If a retrieval of file fails, we would proceed with retrieval
+// of other log files.
+func (w *windowsVM) RetrieveFiles(remoteDir, localDir string) error {
+	if w.sshClient == nil {
+		return fmt.Errorf("RetrieveFile cannot be called without a ssh client")
+	}
+
+	// Create local dir
+	err := os.MkdirAll(localDir, os.ModePerm)
+	if err != nil {
+		log.Printf("could not create %s: %s", localDir, err)
+	}
+
+	sftp, err := sftp.NewClient(w.sshClient)
+	if err != nil {
+		return fmt.Errorf("sftp initialization failed: %v", err)
+	}
+	defer sftp.Close()
+
+	// Get the list of all files in the directory
+	remoteFiles, err := sftp.ReadDir(remoteDir)
+	if err != nil {
+		return fmt.Errorf("error opening remote file: %v", err)
+	}
+
+	for _, remoteFile := range remoteFiles {
+		// Assumption: We ignore the directories here the reason being RetrieveFiles should just retrieve files
+		// in a directory, if this is directory, we should have called RetrieveFiles on this directory
+		if remoteFile.IsDir() {
+			continue
+		}
+		fileName := remoteFile.Name()
+		dstFile, err := os.Create(filepath.Join(localDir, fileName))
+		if err != nil {
+			log.Printf("error creating file locally: %v", err)
+			continue
+		}
+		// TODO: Check if there is some performance implication of multiple Open calls.
+		srcFile, err := sftp.Open(remoteDir + "\\" + fileName)
+
+		if err != nil {
+			log.Printf("error while opening remote directory on the Windows VM: %v", err)
+			continue
+		}
+		_, err = io.Copy(dstFile, srcFile)
+		if err != nil {
+			log.Printf("error retrieving file %v from Windows VM: %v", fileName, err)
+			continue
+		}
+		// flush memory
+		if err = dstFile.Sync(); err != nil {
+			log.Printf("error flusing memory: %v", err)
+			continue
+		}
+		if err := srcFile.Close(); err != nil {
+			log.Printf("error closing file on the remote host %s", fileName)
+			continue
+		}
+		if err := dstFile.Close(); err != nil {
+			log.Printf("error closing file %s locally", fileName)
+			continue
+		}
+	}
+	return nil
+}
+
 func (w *windowsVM) Run(cmd string, psCmd bool) (string, string, error) {
 	if w.winrmClient == nil {
 		return "", "", fmt.Errorf("Run cannot be called without a WinRM client")

internal/test/wmcb/main_test.go

@@ -31,6 +31,8 @@ func TestMain(m *testing.M) {
 		log.Fatal(err)
 	}
 	testStatus := m.Run()
+	// Retrieve artifacts after running the test
+	framework.RetrieveArtifacts()
 	// TODO: Add one more check to remove lingering cloud resources
 	framework.TearDown()
 	os.Exit(testStatus)

internal/test/wsu/main_test.go

@@ -30,6 +30,8 @@ func TestMain(m *testing.M) {
 		log.Fatal(err)
 	}
 	testStatus := m.Run()
+	// Retrieve artifacts after running the test
+	framework.RetrieveArtifacts()
 	// TODO: Add one more check to remove lingering cloud resources
 	framework.TearDown()
 	os.Exit(testStatus)

internal/test/wsu/wsu_test.go

@@ -3,6 +3,7 @@ package wsu
 import (
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"os"
 	"os/exec"
@@ -98,32 +99,44 @@ func testAllVMs(t *testing.T) {
 func setupAndTest(t *testing.T, vmNum int) {
 	// Indicate that we can run the test suite on each node in parallel
 	t.Parallel()
-
+	vm := framework.WinVMs[vmNum]
 	// Run the WSU against the VM
-	wsuOut, err := runWSU(framework.WinVMs[vmNum])
-	require.NoError(t, err, "WSU playbook returned error: %s, with output: %s", err, wsuOut)
-
+	wsuOut, err := runWSU(vm)
+	wsuStringOutput := string(wsuOut)
+	require.NoError(t, err, "WSU playbook returned error: %s, with output: %s", err, wsuStringOutput)
+	// TODO: Think of a better way to refactor this function later to get just output that can be consumed by framework.
+	logFileName := t.Name() + "wsu.log"
+	externalIP := vm.GetCredentials().GetIPAddress()
+	nodeName, err := framework.GetNodeName(externalIP)
+	if err != nil {
+		log.Printf("could not node name associated with the vm %s\n", vm.GetCredentials().GetInstanceId())
+	}
+	require.NoErrorf(t, err, "Error getting node associated with vm %s", vm.GetCredentials().GetInstanceId())
+	localLogDirLocation := filepath.Join("nodes", nodeName, "logs")
+	if err = framework.WriteToArtifactDir(wsuOut, localLogDirLocation, logFileName); err != nil {
+		log.Printf("could not write %s to artifact dir: %s\n", logFileName, err)
+	}
 	// Run our VM test suite
-	runTest(t, framework.WinVMs[vmNum], wsuOut)
+	runTest(t, vm, wsuStringOutput)
 }
 
 // runWSU runs the WSU playbook against a VM. Returns WSU stdout
-func runWSU(vm e2ef.WindowsVM) (string, error) {
+func runWSU(vm e2ef.WindowsVM) ([]byte, error) {
 	var ansibleCmd *exec.Cmd
 
 	// In order to run the ansible playbook we create an inventory file:
 	// https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html
 	hostFilePath, err := createHostFile([]e2ef.WindowsVM{vm})
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	// Run the WSU against the VM
 	ansibleCmd = exec.Command("ansible-playbook", "-v", "-i", hostFilePath, playbookPath)
 
 	// Run the playbook
 	wsuOut, err := ansibleCmd.CombinedOutput()
-	return string(wsuOut), err
+	return wsuOut, err
 }
 
 // testVM runs the WSU against the given VM and runs the e2e test suite against that VM as well