ci: add asup validation check (#1942)

* ci: add asup validation check

integration/test/docker_installer_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/Netapp/harvest-automation/test/installer"
 	"github.com/Netapp/harvest-automation/test/setup"
 	"github.com/Netapp/harvest-automation/test/utils"
+	"strings"
 	"testing"
 )
 
@@ -27,4 +28,28 @@ func TestDockerInstall(t *testing.T) {
 		docker.CopyFile(containerId, fileRestName, installer.HarvestHome+"/"+setup.RestPerfDefaultFile)
 	}
 	docker.ReStartContainers("poller")
+	ids := docker.GetContainerID("poller")
+	if len(ids) > 0 {
+		id := ids[0]
+		if !isValidAsup(id) {
+			panic("Asup validation failed")
+		}
+	} else {
+		panic("No pollers running")
+	}
+
+}
+
+func isValidAsup(containerName string) bool {
+	out, err := utils.Exec("", "docker", nil, "container", "exec", containerName, "autosupport/asup", "--version")
+	if err != nil {
+		fmt.Printf("error %s\n", err)
+		return false
+	}
+	if !strings.Contains(out, "endpoint:stable") {
+		fmt.Printf("asup endpoint is not stable %s\n", out)
+		return false
+	}
+	fmt.Printf("asup validation successful %s\n", out)
+	return true
 }

integration/test/grafana/grafana_mgr.go

@@ -16,6 +16,7 @@ func (g *GrafanaMgr) Import(jsonDir string) (bool, string) {
 	var (
 		importOutput string
 		status       bool
+		err          error
 	)
 	log.Println("Verify Grafana and Prometheus are configured")
 	var re = regexp.MustCompile(`404|not-found|error`)
@@ -34,10 +35,18 @@ func (g *GrafanaMgr) Import(jsonDir string) (bool, string) {
 	if !docker.IsDockerBasedPoller() {
 		//assuming non docker based harvest grafana
 		log.Println("It is non docker based harvest")
-		importOutput = utils.Exec(installer.HarvestHome, "bin/harvest", nil, "grafana", "import", "--addr", utils.GetGrafanaURL(), directoryOption, jsonDir)
+		importOutput, err = utils.Exec(installer.HarvestHome, "bin/harvest", nil, "grafana", "import", "--addr", utils.GetGrafanaURL(), directoryOption, jsonDir)
+		if err != nil {
+			log.Printf("error %s", err)
+			panic(err)
+		}
 	} else {
 		params := []string{"exec", containerIDs[0], "bin/harvest", "grafana", "import", "--addr", "grafana:3000", directoryOption, jsonDir}
-		importOutput = utils.Run("docker", params...)
+		importOutput, err = utils.Run("docker", params...)
+		if err != nil {
+			log.Printf("error %s", err)
+			panic(err)
+		}
 	}
 	if re.MatchString(importOutput) {
 		status = false

integration/test/installer/docker.go

@@ -38,7 +38,11 @@ func (d *Docker) Install() bool {
 			log.Println("Unable to download " + d.path)
 			panic(err)
 		}
-		imageInfo := utils.Run("docker", "load", "-i", tarFileName)
+		imageInfo, err := utils.Run("docker", "load", "-i", tarFileName)
+		if err != nil {
+			log.Printf("error %s", err)
+			panic(err)
+		}
 		imageInfoArray := strings.Split(imageInfo, ":")
 		if len(imageInfoArray) != 3 {
 			panic("docker loaded image has invalid output format")

integration/test/installer/harvest.go

@@ -16,21 +16,33 @@ type Harvest struct {
 }
 
 func (h *Harvest) Start() {
-	status := utils.Exec(HarvestHome, HarvestBin, nil, "start")
+	status, err := utils.Exec(HarvestHome, HarvestBin, nil, "start")
+	if err != nil {
+		log.Printf("error %s", err)
+		panic(err)
+	}
 	fmt.Println(status)
 	time.Sleep(30 * time.Second)
 	h.AllRunning()
 }
 
 func (h *Harvest) StartByHarvestUser() {
-	status := utils.Exec(HarvestHome, "sudo", nil, "-u", "harvest", HarvestBin, "start")
+	status, err := utils.Exec(HarvestHome, "sudo", nil, "-u", "harvest", HarvestBin, "start")
+	if err != nil {
+		log.Printf("error %s", err)
+		panic(err)
+	}
 	fmt.Println(status)
 	time.Sleep(30 * time.Second)
 	h.AllRunning()
 
 }
 func (h *Harvest) Stop() {
-	status := utils.Exec(HarvestHome, HarvestBin, nil, "stop")
+	status, err := utils.Exec(HarvestHome, HarvestBin, nil, "stop")
+	if err != nil {
+		log.Printf("error %s", err)
+		panic(err)
+	}
 	fmt.Println(status)
 }
 
@@ -56,7 +68,11 @@ func (h *Harvest) AllStopped() bool {
 
 func (h *Harvest) GetPollerInfo() []core.Poller {
 	log.Println("Getting all pollers details")
-	harvestStatus := utils.Exec(HarvestHome, HarvestBin, nil, "status")
+	harvestStatus, err := utils.Exec(HarvestHome, HarvestBin, nil, "status")
+	if err != nil {
+		log.Printf("error %s", err)
+		panic(err)
+	}
 	fmt.Println(harvestStatus)
 	rows := strings.Split(harvestStatus, "\n")
 	var pollerArray []core.Poller
@@ -82,3 +98,17 @@ func (h *Harvest) GetPollerInfo() []core.Poller {
 	}
 	return pollerArray
 }
+
+func (h *Harvest) IsValidAsup(asupExecPath string) bool {
+	out, err := utils.Exec("", asupExecPath, nil, "--version")
+	if err != nil {
+		fmt.Printf("error %s\n", err)
+		return false
+	}
+	if !strings.Contains(out, "endpoint:stable") {
+		fmt.Printf("asup endpoint is not stable %s\n", out)
+		return false
+	}
+	fmt.Printf("asup validation successful %s\n", out)
+	return true
+}

integration/test/installer/install_utils.go

@@ -24,8 +24,10 @@ func UninstallNativePkg() {
 	log.Println("Uninstalling  native pkg if any")
 	if utils.FileExists(HarvestHome) {
 		harvestObj := new(Harvest)
-		if harvestObj.AllRunning() {
-			harvestObj.Stop()
+		if utils.FileExists(HarvestHome + "/bin/harvest") {
+			if harvestObj.AllRunning() {
+				harvestObj.Stop()
+			}
 		}
 		utils.Run("rm", "-rf", HarvestHome)
 	} else {

integration/test/installer/native.go

@@ -27,7 +27,11 @@ func (n *Native) Install() bool {
 	log.Println("Downloaded: " + n.path)
 	Uninstall()
 	log.Println("Installing " + tarFileName)
-	unTarOutput := utils.Run("tar", "-xf", tarFileName, "--one-top-level=harvest", "--strip-components", "1", "-C", "/opt")
+	unTarOutput, err := utils.Run("tar", "-xf", tarFileName, "--one-top-level=harvest", "--strip-components", "1", "-C", "/opt")
+	if err != nil {
+		log.Printf("error %s", err)
+		panic(err)
+	}
 	log.Println(unTarOutput)
 	utils.RemoveSafely(HarvestHome + "/" + harvestFile)
 	utils.UseCertFile(HarvestHome)
@@ -39,7 +43,9 @@ func (n *Native) Install() bool {
 	}
 	harvestObj.Start()
 	status := harvestObj.AllRunning()
-	return status
+	asupExecPath := HarvestHome + "/autosupport/asup"
+	isValidAsup := harvestObj.IsValidAsup(asupExecPath)
+	return status && isValidAsup
 }
 
 func (n *Native) Upgrade() bool {

integration/test/installer/rpm.go

@@ -28,7 +28,11 @@ func (r *RPM) Install() bool {
 	Uninstall()
 	harvestObj := new(Harvest)
 	log.Println("Installing " + rpmFileName)
-	installOutput := utils.Run("yum", "install", "-y", rpmFileName)
+	installOutput, err := utils.Run("yum", "install", "-y", rpmFileName)
+	if err != nil {
+		log.Printf("error %s", err)
+		panic(err)
+	}
 	log.Println(installOutput)
 	log.Println("Stopping harvest")
 	harvestObj.Stop()
@@ -38,6 +42,9 @@ func (r *RPM) Install() bool {
 	} //use file directly from the repo
 	harvestObj.Start()
 	status := harvestObj.AllRunning()
+	asupExecPath := HarvestHome + "/autosupport/asup"
+	isValidAsup := harvestObj.IsValidAsup(asupExecPath)
+	return status && isValidAsup
 	return status
 }
 
@@ -49,14 +56,20 @@ func (r *RPM) Upgrade() bool {
 		utils.PanicIfNotNil(fmt.Errorf("pollers are not in a running state before upgrade"))
 	}
 	versionCmd := []string{"-qa", "harvest"}
-	previousVersion := strings.TrimSpace(utils.Run("rpm", versionCmd...))
-	err := utils.DownloadFile(rpmFileName, r.path)
+	out, err := utils.Run("rpm", versionCmd...)
+	if err != nil {
+		log.Printf("error %s", err)
+		panic(err)
+	}
+	previousVersion := strings.TrimSpace(out)
+	err = utils.DownloadFile(rpmFileName, r.path)
 	utils.PanicIfNotNil(err)
 	log.Println("Downloaded: " + r.path)
 	log.Println("Updating " + rpmFileName)
-	installOutput := utils.Run("yum", "upgrade", "-y", rpmFileName)
+	installOutput, _ := utils.Run("yum", "upgrade", "-y", rpmFileName)
 	log.Println(installOutput)
-	installedVersion := strings.TrimSpace(utils.Run("rpm", versionCmd...))
+	out, _ = utils.Run("rpm", versionCmd...)
+	installedVersion := strings.TrimSpace(out)
 	if previousVersion == installedVersion {
 		utils.PanicIfNotNil(fmt.Errorf("upgrade is failed"))
 	}
@@ -65,5 +78,7 @@ func (r *RPM) Upgrade() bool {
 	harvestObj.Stop()
 	harvestObj.Start()
 	status := harvestObj.AllRunning()
-	return status
+	asupExecPath := HarvestHome + "/autosupport/asup"
+	isValidAsup := harvestObj.IsValidAsup(asupExecPath)
+	return status && isValidAsup
 }

integration/test/utils/utils.go

@@ -24,7 +24,7 @@ const (
 	GrafanaTokeKey = "grafana_api_token"
 )
 
-func Run(command string, arg ...string) string {
+func Run(command string, arg ...string) (string, error) {
 	return Exec("", command, nil, arg...)
 }
 
@@ -43,7 +43,7 @@ func GetConfigDir() string {
 	return "/u/mpeg/harvest"
 }
 
-func Exec(dir string, command string, env []string, arg ...string) string {
+func Exec(dir string, command string, env []string, arg ...string) (string, error) {
 	cmdString := command + " "
 	for _, param := range arg {
 		cmdString = cmdString + param + " "
@@ -70,7 +70,7 @@ func Exec(dir string, command string, env []string, arg ...string) string {
 		fmt.Println(err)
 	}
 	fmt.Println("-------------------------")
-	return out.String()
+	return out.String(), err
 }
 
 // DownloadFile will download a url to a local file. It's efficient because it will