MGMT-14633: WIP include manifest information in the log download

internal/cluster/cluster.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"path/filepath"
 	"reflect"
@@ -40,6 +41,7 @@ import (
 	"github.com/openshift/assisted-service/pkg/ocm"
 	"github.com/openshift/assisted-service/pkg/requestid"
 	"github.com/openshift/assisted-service/pkg/s3wrapper"
+	operations "github.com/openshift/assisted-service/restapi/operations/manifests"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/thoas/go-funk"
@@ -1105,7 +1107,45 @@ func (m *Manager) createClusterDataFiles(ctx context.Context, c *common.Cluster,
 		fileName := fmt.Sprintf("%s/logs/cluster/events.json", c.ID)
 		_ = m.uploadDataAsFile(ctx, log, events, fileName, objectHandler)
 	}
+
+	includeSystemGenerated := true
+	listManifests, _ := m.manifestApi.ListClusterManifestsInternal(ctx, operations.V2ListClusterManifestsParams{ClusterID: *c.ID, IncludeSystemGenerated: &includeSystemGenerated})
+	for _, manifestEntry := range listManifests {
+		// Download the manifest content
+		path := filepath.Join(manifestEntry.Folder, manifestEntry.FileName)
+		objectName := filepath.Join(string(*c.ID), constants.ManifestFolder, path)
+		exists, _ := objectHandler.DoesObjectExist(ctx, objectName)
+		if !exists {
+			continue
+		}
+		response, numberOfBytes, err := m.objectHandler.Download(ctx, objectName)
+		if err != nil {
+			log.WithError(err).Errorf("failed to download file %s from cluster while building log: %s", path, *c.ID)
+			continue
+		}
+		fileContent, err := io.ReadAll(response)
+		if err != nil {
+			log.WithError(err).Errorf("failed to read file data %s (%d) from cluster while building log: %s", path, numberOfBytes, *c.ID)
+			continue
+		}
+		// Determine the manifest type, this will determine where the file is uploaded to
+		isUserManifest, err := m.manifestApi.IsUserManifest(ctx, *c.ID, manifestEntry.Folder, manifestEntry.FileName)
+		manifest_prefix := "system-generated"
+		if err != nil {
+			log.WithError(err).Errorf("Unable to determine if manifest is user generated or not for %s/%s", manifestEntry.Folder, manifestEntry.FileName)
+			manifest_prefix = "unknown"
+		}
+		if isUserManifest {
+			manifest_prefix = "user-supplied"
+		}
+		fileName := fmt.Sprintf("%s/logs/manifest_%s_%s/%s", c.ID, manifest_prefix, manifestEntry.Folder, manifestEntry.FileName)
+		err = m.uploadDataAsFile(ctx, log, string(fileContent), fileName, objectHandler)
+		if err != nil {
+			log.WithError(err).Errorf("Unable to upload %s to logs for cluster id %s", fileName, *c.ID)
+		}
+	}
 }
+
 func (m *Manager) PrepareHostLogFile(ctx context.Context, c *common.Cluster, host *models.Host, objectHandler s3wrapper.API) (string, error) {
 	var (
 		fileName        string
@@ -1192,11 +1232,13 @@ func (m *Manager) PrepareClusterLogFile(ctx context.Context, c *common.Cluster,
 
 	for _, file := range allFiles {
 		fileNameSplit := strings.Split(file, "/")
+
 		if len(fileNameSplit) < 2 {
 			selectedFiles = append(selectedFiles, file)
 			tarredFilenames = append(tarredFilenames, file)
 			continue
 		}
+
 		hostId := getHostIdFromPath(fileNameSplit)
 		if hostId == nil {
 			tarredFilename = fmt.Sprintf("%s_%s", fileNameSplit[len(fileNameSplit)-2], fileNameSplit[len(fileNameSplit)-1])

internal/cluster/cluster_test.go

@@ -2783,6 +2783,7 @@ var _ = Describe("Cluster tarred files", func() {
 		tarFile               string
 		clusterObjectFilename string
 		eventsFilename        string
+		mockManifestApi       *manifestsapi.MockManifestsAPI
 	)
 
 	uploadClusterDataSuccess := func() {
@@ -2808,7 +2809,8 @@ var _ = Describe("Cluster tarred files", func() {
 		mockEvents = eventsapi.NewMockHandler(ctrl)
 		dummy := &leader.DummyElector{}
 		mockOperators := operators.NewMockAPI(ctrl)
-		capi = NewManager(cfg, common.GetTestLog(), db, commontesting.GetDummyNotificationStream(ctrl), mockEvents, nil, mockHostAPI, nil, nil, dummy, mockOperators, nil, nil, nil, nil, nil)
+		mockManifestApi = manifestsapi.NewMockManifestsAPI(ctrl)
+		capi = NewManager(cfg, common.GetTestLog(), db, commontesting.GetDummyNotificationStream(ctrl), mockEvents, nil, mockHostAPI, nil, nil, dummy, mockOperators, nil, nil, nil, nil, mockManifestApi)
 		clusterId = strfmt.UUID(uuid.New().String())
 		cl = common.Cluster{
 			Cluster: models.Cluster{
@@ -2824,7 +2826,7 @@ var _ = Describe("Cluster tarred files", func() {
 		prefix = fmt.Sprintf("%s/logs/", cl.ID)
 		mockEvents.EXPECT().SendClusterEvent(gomock.Any(), eventstest.NewEventMatcher(
 			eventstest.WithClusterIdMatcher(clusterId.String()))).AnyTimes()
-
+		mockManifestApi.EXPECT().ListClusterManifestsInternal(gomock.Any(), gomock.Any()).Return(models.ListManifests{}, nil)
 	})
 
 	AfterEach(func() {

subsystem/cluster_test.go

@@ -2390,7 +2390,32 @@ var _ = Describe("cluster install", func() {
 			}
 		})
 
+		uploadManifest := func(content string, folder string, filename string) {
+			base64Content := base64.StdEncoding.EncodeToString([]byte(content))
+			response, err := userBMClient.Manifests.V2CreateClusterManifest(ctx, &manifests.V2CreateClusterManifestParams{
+				ClusterID: clusterID,
+				CreateManifestParams: &models.CreateManifestParams{
+					Content:  &base64Content,
+					FileName: &filename,
+					Folder:   &folder,
+				},
+			})
+			Expect(err).ShouldNot(HaveOccurred())
+			Expect(*response.Payload).Should(Not(BeNil()))
+		}
+
+		manifest1Content := `apiVersion: v1
+kind: Namespace
+metadata:
+name: exampleNamespace1`
+
+		manifest2Content := `apiVersion: v1
+kind: Namespace
+metadata:
+name: exampleNamespace2`
+
 		It("Download cluster logs", func() {
+			// Add some manifest files and then verify that these are added to the log...
 			nodes, _ := register3nodes(ctx, clusterID, *infraEnvID, defaultCIDRv4)
 			for _, host := range nodes {
 				kubeconfigFile, err := os.Open("test_kubeconfig")
@@ -2407,6 +2432,9 @@ var _ = Describe("cluster install", func() {
 			Expect(err).NotTo(HaveOccurred())
 			kubeconfigFile.Close()
 
+			uploadManifest(manifest1Content, "openshift", "manifest1.yaml")
+			uploadManifest(manifest2Content, "openshift", "manifest2.yaml")
+
 			filePath := "../build/test_logs.tar"
 			file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
 			Expect(err).NotTo(HaveOccurred())
@@ -2422,17 +2450,17 @@ var _ = Describe("cluster install", func() {
 			Expect(err).NotTo(HaveOccurred())
 			tarReader := tar.NewReader(file)
 			numOfarchivedFiles := 0
+			expectedFiles := []string{"manifest_user-supplied_openshift_manifest1.yaml", "manifest_user-supplied_openshift_manifest2.yaml", "cluster_events.json", "cluster_metadata.json", "controller_logs.tar.gz", "test-cluster_auto-assign_h1.tar", "test-cluster_auto-assign_h2.tar", "test-cluster_auto-assign_h3.tar"}
 			for {
-				_, err := tarReader.Next()
+				header, err := tarReader.Next()
 				if err == io.EOF {
 					break
 				}
+				Expect(swag.ContainsStrings(expectedFiles, header.Name)).To(BeTrue())
 				Expect(err).NotTo(HaveOccurred())
 				numOfarchivedFiles += 1
-				Expect(numOfarchivedFiles <= len(nodes)+3).Should(Equal(true))
 			}
-			Expect(numOfarchivedFiles).Should(Equal(len(nodes) + 3))
-
+			Expect(numOfarchivedFiles).Should(Equal(len(expectedFiles)))
 		})
 
 		It("Upload ingress ca and kubeconfig download", func() {