Skip to content

Commit

Permalink
MGMT-13454: Group Host and boot logs to a single tarball
Browse files Browse the repository at this point in the history 
Several changes in how we package logs:
1. Host logs will include host and boot logs under a single file.
2. Cluster logs will now include the host tar.gz file
   containing the content mentioned in openshift#1.
3. The option to download node-boot logs only is disabled.
  • Loading branch information
Nir Magnezi committed Feb 13, 2023
1 parent ef2761d commit 00daeb8
Show file tree
Hide file tree
Showing 8 changed files with 137 additions and 251 deletions.
26 changes: 16 additions & 10 deletions internal/bminventory/inventory.go
View file
Expand Up @@ -3565,35 +3565,35 @@ func (b *bareMetalInventory) DownloadMinimalInitrd(ctx context.Context, params i
func (b *bareMetalInventory) getLogFileForDownload(ctx context.Context, clusterId *strfmt.UUID, hostId *strfmt.UUID, logsType string) (string, string, error) {
var fileName string
var downloadFileName string
var hostObject *common.Host

c, err := b.getCluster(ctx, clusterId.String(), common.UseEagerLoading, common.IncludeDeletedRecords)
if err != nil {
return "", "", err
}
b.log.Debugf("log type to download: %s", logsType)
switch logsType {
case string(models.LogsTypeHost), string(models.LogsTypeNodeBoot):
case string(models.LogsTypeHost):
if hostId == nil {
return "", "", common.NewApiError(http.StatusBadRequest, errors.Errorf("Host ID must be provided for downloading host logs"))
}

var hostObject *common.Host
hostObject, err = common.GetClusterHostFromDB(b.db, clusterId.String(), hostId.String())
if err != nil {
return "", "", err
}
if time.Time(hostObject.LogsCollectedAt).Equal(time.Time{}) {
return "", "", common.NewApiError(http.StatusConflict, errors.Errorf("Logs for host %s were not found", hostId))
}
fileName = b.getLogsFullName(logsType, clusterId.String(), hostObject.ID.String())
role := string(hostObject.Role)
if hostObject.Bootstrap {
role = string(models.HostRoleBootstrap)
}
name := sanitize.Name(hostutil.GetHostnameForMsg(&hostObject.Host))
if logsType == string(models.LogsTypeNodeBoot) {
name = fmt.Sprintf("boot_%s", name)
}
downloadFileName = fmt.Sprintf("%s_%s_%s.tar.gz", sanitize.Name(c.Name), role, name)
fileName, err = b.preparHostLogs(ctx, c, &hostObject.Host)
if err != nil {
return "", "", err
}
case string(models.LogsTypeController):
if time.Time(c.Cluster.ControllerLogsCollectedAt).Equal(time.Time{}) {
return "", "", common.NewApiError(http.StatusConflict, errors.Errorf("Controller Logs for cluster %s were not found", clusterId))
Expand Down Expand Up @@ -3968,9 +3968,7 @@ func (b *bareMetalInventory) uploadHostLogs(ctx context.Context, host *common.Ho
log.WithError(err).Errorf("Failed update host %s logs_collected_at flag", host.ID.String())
return common.NewApiError(http.StatusInternalServerError, err)
}
if logsType == string(models.LogsTypeNodeBoot) {
return nil
}

err = b.hostApi.UpdateLogsProgress(ctx, &host.Host, string(models.LogsStateCollecting))
if err != nil {
log.WithError(err).Errorf("Failed update host %s log progress %s", host.ID.String(), string(models.LogsStateCollecting))
Expand All @@ -3987,6 +3985,14 @@ func (b *bareMetalInventory) prepareClusterLogs(ctx context.Context, cluster *co
return fileName, nil
}

func (b *bareMetalInventory) preparHostLogs(ctx context.Context, cluster *common.Cluster, host *models.Host) (string, error) {
fileName, err := b.clusterApi.PrepareHostLogFile(ctx, cluster, host, b.objectHandler)
if err != nil {
return "", err
}
return fileName, nil
}

func (b *bareMetalInventory) getLogsFullName(logType string, clusterId string, logId string) string {
filename := "logs.tar.gz"
if logType == string(models.LogsTypeNodeBoot) {
Expand Down
187 changes: 6 additions & 181 deletions internal/bminventory/inventory_test.go
View file
Expand Up @@ -9606,19 +9606,6 @@ var _ = Describe("Upload and Download logs test", func() {
verifyApiError(bm.V2UploadLogs(ctx, params), http.StatusNotFound)
})

It("Upload boot logs host not exits", func() {
hostId := strToUUID(uuid.New().String())
params := installer.V2UploadLogsParams{
ClusterID: clusterID,
HostID: hostId,
LogsType: string(models.LogsTypeNodeBoot),
InfraEnvID: &clusterID,
Upfile: kubeconfigFile,
HTTPRequest: request,
}
verifyApiError(bm.V2UploadLogs(ctx, params), http.StatusNotFound)
})

It("Upload host logs to S3 - upload fails", func() {
newHostID := strfmt.UUID(uuid.New().String())
host := addHost(newHostID, models.HostRoleMaster, "known", models.HostKindHost, clusterID, clusterID, "{}", db)
Expand All @@ -9636,23 +9623,6 @@ var _ = Describe("Upload and Download logs test", func() {
verifyApiError(bm.V2UploadLogs(ctx, params), http.StatusInternalServerError)
})

It("Upload host boot logs to S3 - upload fails", func() {
newHostID := strfmt.UUID(uuid.New().String())
host := addHost(newHostID, models.HostRoleMaster, "known", models.HostKindHost, clusterID, clusterID, "{}", db)
params := installer.V2UploadLogsParams{
ClusterID: clusterID,
HostID: host.ID,
LogsType: string(models.LogsTypeNodeBoot),
InfraEnvID: &clusterID,
Upfile: kubeconfigFile,
HTTPRequest: request,
}

fileName := bm.getLogsFullName(string(models.LogsTypeNodeBoot), clusterID.String(), host.ID.String())
mockS3Client.EXPECT().UploadStream(gomock.Any(), gomock.Any(), fileName).Return(errors.Errorf("Dummy")).Times(1)
verifyApiError(bm.V2UploadLogs(ctx, params), http.StatusInternalServerError)
})

It("Upload Hosts logs Happy flow", func() {

newHostID := strfmt.UUID(uuid.New().String())
Expand All @@ -9677,29 +9647,6 @@ var _ = Describe("Upload and Download logs test", func() {
Expect(reply).Should(BeAssignableToTypeOf(installer.NewV2UploadLogsNoContent()))
})

It("Upload Hosts boot logs Happy flow", func() {

newHostID := strfmt.UUID(uuid.New().String())
host := addHost(newHostID, models.HostRoleMaster, "known", models.HostKindHost, clusterID, clusterID, "{}", db)
params := installer.V2UploadLogsParams{
ClusterID: clusterID,
HostID: host.ID,
LogsType: string(models.LogsTypeNodeBoot),
InfraEnvID: &clusterID,
Upfile: kubeconfigFile,
HTTPRequest: request,
}
fileName := bm.getLogsFullName(string(models.LogsTypeNodeBoot), clusterID.String(), host.ID.String())
mockEvents.EXPECT().SendHostEvent(gomock.Any(), eventstest.NewEventMatcher(
eventstest.WithNameMatcher(eventgen.HostBootLogsUploadedEventName),
eventstest.WithClusterIdMatcher(clusterID.String()),
eventstest.WithHostIdMatcher(host.ID.String()))).Times(1)
mockS3Client.EXPECT().UploadStream(gomock.Any(), gomock.Any(), fileName).Return(nil).Times(1)
mockHostApi.EXPECT().SetUploadLogsAt(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)
reply := bm.V2UploadLogs(ctx, params)
Expect(reply).Should(BeAssignableToTypeOf(installer.NewV2UploadLogsNoContent()))
})

It(" Start collecting hosts logs indication", func() {
newHostID := strfmt.UUID(uuid.New().String())
host := addHost(newHostID, models.HostRoleMaster, "known", models.HostKindHost, infraEnvID, clusterID, "{}", db)
Expand Down Expand Up @@ -9764,17 +9711,6 @@ var _ = Describe("Upload and Download logs test", func() {
verifyApiError(bm.V2DownloadClusterLogs(ctx, params), http.StatusConflict)
})

It("Download S3 host boot logs where not uploaded yet", func() {
logsType := string(models.LogsTypeNodeBoot)
params := installer.V2DownloadClusterLogsParams{
ClusterID: clusterID,
HostID: &hostID,
LogsType: &logsType,
HTTPRequest: request,
}
verifyApiError(bm.V2DownloadClusterLogs(ctx, params), http.StatusConflict)
})

It("Download S3 host logs where not uploaded yet", func() {
logsType := string(models.LogsTypeHost)
params := installer.V2DownloadClusterLogsParams{
Expand All @@ -9786,21 +9722,6 @@ var _ = Describe("Upload and Download logs test", func() {
verifyApiError(bm.V2DownloadClusterLogs(ctx, params), http.StatusConflict)
})

It("Download S3 host boot logs - object not found", func() {
logsType := string(models.LogsTypeNodeBoot)
params := installer.V2DownloadClusterLogsParams{
ClusterID: clusterID,
HostID: &hostID,
LogsType: &logsType,
HTTPRequest: request,
}
host1.LogsCollectedAt = strfmt.DateTime(time.Now())
db.Save(&host1)
fileName := bm.getLogsFullName(logsType, clusterID.String(), hostID.String())
mockS3Client.EXPECT().Download(ctx, fileName).Return(nil, int64(0), common.NotFound(fileName))
verifyApiError(bm.V2DownloadClusterLogs(ctx, params), http.StatusNotFound)
})

It("Download S3 host logs - object not found", func() {
logsType := string(models.LogsTypeHost)
params := installer.V2DownloadClusterLogsParams{
Expand All @@ -9813,24 +9734,10 @@ var _ = Describe("Upload and Download logs test", func() {
db.Save(&host1)
fileName := bm.getLogsFullName(logsType, clusterID.String(), hostID.String())
mockS3Client.EXPECT().Download(ctx, fileName).Return(nil, int64(0), common.NotFound(fileName))
mockClusterApi.EXPECT().PrepareHostLogFile(ctx, gomock.Any(), gomock.Any(), gomock.Any()).Return(fileName, nil).Times(1)
verifyApiError(bm.V2DownloadClusterLogs(ctx, params), http.StatusNotFound)
})

It("Download S3 host boot logs - object failed", func() {
logsType := string(models.LogsTypeNodeBoot)
params := installer.V2DownloadClusterLogsParams{
ClusterID: clusterID,
HostID: &hostID,
LogsType: &logsType,
HTTPRequest: request,
}
host1.LogsCollectedAt = strfmt.DateTime(time.Now())
db.Save(&host1)
fileName := bm.getLogsFullName(logsType, clusterID.String(), hostID.String())
mockS3Client.EXPECT().Download(ctx, fileName).Return(nil, int64(0), errors.Errorf("dummy"))
verifyApiError(bm.V2DownloadClusterLogs(ctx, params), http.StatusInternalServerError)
})

It("Download S3 host logs -object failed", func() {
logsType := string(models.LogsTypeHost)
params := installer.V2DownloadClusterLogsParams{
Expand All @@ -9842,6 +9749,7 @@ var _ = Describe("Upload and Download logs test", func() {
host1.LogsCollectedAt = strfmt.DateTime(time.Now())
db.Save(&host1)
fileName := bm.getLogsFullName(logsType, clusterID.String(), hostID.String())
mockClusterApi.EXPECT().PrepareHostLogFile(ctx, gomock.Any(), gomock.Any(), gomock.Any()).Return(fileName, nil).Times(1)
mockS3Client.EXPECT().Download(ctx, fileName).Return(nil, int64(0), errors.Errorf("dummy"))
verifyApiError(bm.V2DownloadClusterLogs(ctx, params), http.StatusInternalServerError)
})
Expand All @@ -9861,33 +9769,13 @@ var _ = Describe("Upload and Download logs test", func() {
host.LogsCollectedAt = strfmt.DateTime(time.Now())
db.Save(&host)
r := io.NopCloser(bytes.NewReader([]byte("test")))
mockClusterApi.EXPECT().PrepareHostLogFile(ctx, gomock.Any(), gomock.Any(), gomock.Any()).Return(fileName, nil).Times(1)
mockS3Client.EXPECT().Download(ctx, fileName).Return(r, int64(4), nil)
generateReply := bm.V2DownloadClusterLogs(ctx, params)
downloadFileName := fmt.Sprintf("mycluster_bootstrap_%s.tar.gz", newHostID.String())
Expect(generateReply).Should(Equal(filemiddleware.NewResponder(installer.NewV2DownloadClusterLogsOK().WithPayload(r), downloadFileName, 4, nil)))
})

It("Download Hosts boot logs happy flow", func() {
newHostID := strfmt.UUID(uuid.New().String())
host := addHost(newHostID, models.HostRoleMaster, "known", models.HostKindHost, clusterID, clusterID, "{}", db)
logsType := string(models.LogsTypeNodeBoot)
params := installer.V2DownloadClusterLogsParams{
ClusterID: clusterID,
HostID: host.ID,
LogsType: &logsType,
HTTPRequest: request,
}
host.Bootstrap = true
fileName := bm.getLogsFullName(logsType, clusterID.String(), host.ID.String())
host.LogsCollectedAt = strfmt.DateTime(time.Now())
db.Save(&host)
r := io.NopCloser(bytes.NewReader([]byte("test")))
mockS3Client.EXPECT().Download(ctx, fileName).Return(r, int64(4), nil)
generateReply := bm.V2DownloadClusterLogs(ctx, params)
downloadFileName := fmt.Sprintf("mycluster_bootstrap_boot_%s.tar.gz", newHostID.String())
Expect(generateReply).Should(Equal(filemiddleware.NewResponder(installer.NewV2DownloadClusterLogsOK().WithPayload(r), downloadFileName, 4, nil)))
})

It("Download Controller logs happy flow", func() {
logsType := string(models.LogsTypeController)
params := installer.V2DownloadClusterLogsParams{
Expand Down Expand Up @@ -9916,19 +9804,6 @@ var _ = Describe("Upload and Download logs test", func() {
verifyApiError(generateReply, http.StatusNotFound)
})

It("Host boot logs presigned host not found", func() {
logsType := string(models.LogsTypeNodeBoot)
hostID := strfmt.UUID(uuid.New().String())
mockS3Client.EXPECT().IsAwsS3().Return(true)
generateReply := bm.V2GetPresignedForClusterFiles(ctx, installer.V2GetPresignedForClusterFilesParams{
ClusterID: clusterID,
FileName: "logs",
HostID: &hostID,
LogsType: &logsType,
})
verifyApiError(generateReply, http.StatusNotFound)
})

It("Host logs presigned no logs found", func() {
hostID := strfmt.UUID(uuid.New().String())
_ = addHost(hostID, models.HostRoleMaster, "known", models.HostKindHost, clusterID, clusterID, "{}", db)
Expand All @@ -9942,25 +9817,12 @@ var _ = Describe("Upload and Download logs test", func() {
verifyApiError(generateReply, http.StatusConflict)
})

It("Host boot logs presigned no logs found", func() {
logsType := string(models.LogsTypeNodeBoot)
hostID := strfmt.UUID(uuid.New().String())
_ = addHost(hostID, models.HostRoleMaster, "known", models.HostKindHost, clusterID, clusterID, "{}", db)
mockS3Client.EXPECT().IsAwsS3().Return(true)
generateReply := bm.V2GetPresignedForClusterFiles(ctx, installer.V2GetPresignedForClusterFilesParams{
ClusterID: clusterID,
FileName: "logs",
HostID: &hostID,
LogsType: &logsType,
})
verifyApiError(generateReply, http.StatusConflict)
})

It("Host logs presigned s3 error", func() {
hostID := strfmt.UUID(uuid.New().String())
host1 = addHost(hostID, models.HostRoleMaster, "known", models.HostKindHost, clusterID, clusterID, "{}", db)
mockS3Client.EXPECT().IsAwsS3().Return(true)
fileName := bm.getLogsFullName(string(models.LogsTypeHost), clusterID.String(), hostID.String())
mockClusterApi.EXPECT().PrepareHostLogFile(ctx, gomock.Any(), gomock.Any(), gomock.Any()).Return(fileName, nil).Times(1)
host1.LogsCollectedAt = strfmt.DateTime(time.Now())
db.Save(&host1)
mockS3Client.EXPECT().GeneratePresignedDownloadURL(ctx, fileName, fmt.Sprintf("mycluster_master_%s.tar.gz", hostID.String()), gomock.Any()).Return("",
Expand All @@ -9974,30 +9836,12 @@ var _ = Describe("Upload and Download logs test", func() {
verifyApiError(generateReply, http.StatusInternalServerError)
})

It("Host boot logs presigned s3 error", func() {
logsType := string(models.LogsTypeNodeBoot)
hostID := strfmt.UUID(uuid.New().String())
host1 = addHost(hostID, models.HostRoleMaster, "known", models.HostKindHost, clusterID, clusterID, "{}", db)
mockS3Client.EXPECT().IsAwsS3().Return(true)
fileName := bm.getLogsFullName(string(models.LogsTypeNodeBoot), clusterID.String(), hostID.String())
host1.LogsCollectedAt = strfmt.DateTime(time.Now())
db.Save(&host1)
mockS3Client.EXPECT().GeneratePresignedDownloadURL(ctx, fileName, fmt.Sprintf("mycluster_master_boot_%s.tar.gz", hostID.String()), gomock.Any()).Return("",
errors.Errorf("Dummy"))
generateReply := bm.V2GetPresignedForClusterFiles(ctx, installer.V2GetPresignedForClusterFilesParams{
ClusterID: clusterID,
FileName: "logs",
HostID: &hostID,
LogsType: &logsType,
})
verifyApiError(generateReply, http.StatusInternalServerError)
})

It("host logs presigned happy flow", func() {
hostID := strfmt.UUID(uuid.New().String())
host1 = addHost(hostID, models.HostRoleMaster, "known", models.HostKindHost, clusterID, clusterID, "{}", db)
mockS3Client.EXPECT().IsAwsS3().Return(true)
fileName := bm.getLogsFullName(string(models.LogsTypeHost), clusterID.String(), hostID.String())
mockClusterApi.EXPECT().PrepareHostLogFile(ctx, gomock.Any(), gomock.Any(), gomock.Any()).Return(fileName, nil).Times(1)
host1.LogsCollectedAt = strfmt.DateTime(time.Now())
db.Save(&host1)
mockS3Client.EXPECT().GeneratePresignedDownloadURL(ctx, fileName, fmt.Sprintf("mycluster_master_%s.tar.gz", hostID.String()), gomock.Any()).Return("url", nil)
Expand All @@ -10012,31 +9856,12 @@ var _ = Describe("Upload and Download logs test", func() {
Expect(*replyPayload.URL).Should(Equal("url"))
})

It("host boot logs presigned happy flow", func() {
logsType := string(models.LogsTypeNodeBoot)
hostID := strfmt.UUID(uuid.New().String())
host1 = addHost(hostID, models.HostRoleMaster, "known", models.HostKindHost, clusterID, clusterID, "{}", db)
mockS3Client.EXPECT().IsAwsS3().Return(true)
fileName := bm.getLogsFullName(string(models.LogsTypeNodeBoot), clusterID.String(), hostID.String())
host1.LogsCollectedAt = strfmt.DateTime(time.Now())
db.Save(&host1)
mockS3Client.EXPECT().GeneratePresignedDownloadURL(ctx, fileName, fmt.Sprintf("mycluster_master_boot_%s.tar.gz", hostID.String()), gomock.Any()).Return("url", nil)
generateReply := bm.V2GetPresignedForClusterFiles(ctx, installer.V2GetPresignedForClusterFilesParams{
ClusterID: clusterID,
FileName: "logs",
HostID: &hostID,
LogsType: &logsType,
})
Expect(generateReply).Should(BeAssignableToTypeOf(&installer.V2GetPresignedForClusterFilesOK{}))
replyPayload := generateReply.(*installer.V2GetPresignedForClusterFilesOK).Payload
Expect(*replyPayload.URL).Should(Equal("url"))
})

It("host logs presigned happy flow without log type", func() {
hostID := strfmt.UUID(uuid.New().String())
host1 = addHost(hostID, models.HostRoleMaster, "known", models.HostKindHost, clusterID, clusterID, "{}", db)
mockS3Client.EXPECT().IsAwsS3().Return(true)
fileName := bm.getLogsFullName(string(models.LogsTypeHost), clusterID.String(), hostID.String())
mockClusterApi.EXPECT().PrepareHostLogFile(ctx, gomock.Any(), gomock.Any(), gomock.Any()).Return(fileName, nil).Times(1)
host1.LogsCollectedAt = strfmt.DateTime(time.Now())
db.Save(&host1)
mockS3Client.EXPECT().GeneratePresignedDownloadURL(ctx, fileName, fmt.Sprintf("mycluster_master_%s.tar.gz", hostID.String()), gomock.Any()).Return("url", nil)
Expand Down

0 comments on commit 00daeb8

Please sign in to comment.