diff --git a/Dockerfile.assisted-service b/Dockerfile.assisted-service index ec7645b8084..c050c9891c4 100644 --- a/Dockerfile.assisted-service +++ b/Dockerfile.assisted-service @@ -38,6 +38,8 @@ RUN cd ./cmd/operator && CGO_ENABLED=1 GOFLAGS="" GO111MODULE=on go build -o /bu RUN cd ./cmd/webadmission && CGO_ENABLED=1 GOFLAGS="" GO111MODULE=on go build -o /build/assisted-service-admission RUN cd ./cmd/agentbasedinstaller/client && CGO_ENABLED=1 GOFLAGS="" GO111MODULE=on go build -o /build/agent-installer-client +COPY internal/ignition/boot-reporter/assisted-boot-reporter.sh /build + # Create final image FROM quay.io/centos/centos:stream8 @@ -58,6 +60,7 @@ COPY --from=builder /build/assisted-service /assisted-service COPY --from=builder /build/assisted-service-operator /assisted-service-operator COPY --from=builder /build/assisted-service-admission /assisted-service-admission COPY --from=builder /build/agent-installer-client /usr/local/bin/agent-installer-client +COPY --from=builder /build/assisted-boot-reporter.sh /assisted-boot-reporter.sh RUN ln -s /usr/local/bin/agent-installer-client /agent-based-installer-register-cluster-and-infraenv COPY --from=pybuilder /assisted-service/build/dist/* /clients/ ENV GODEBUG=madvdontneed=1 diff --git a/cmd/main.go b/cmd/main.go index 3572664199e..1646fd98f61 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -390,7 +390,7 @@ func main() { failOnError(err, "failed to create valid bm config S3 endpoint URL from %s", Options.BMConfig.S3EndpointURL) Options.BMConfig.S3EndpointURL = newUrl - generator := generator.New(log, objectHandler, Options.GeneratorConfig, Options.WorkDir, operatorsManager, providerRegistry, Options.ClusterTLSCertOverrideDir) + generator := generator.New(log, db, objectHandler, Options.GeneratorConfig, Options.WorkDir, operatorsManager, providerRegistry, Options.ClusterTLSCertOverrideDir) var crdUtils bminventory.CRDUtils if ctrlMgr != nil { crdUtils = controllers.NewCRDUtils(ctrlMgr.GetClient(), hostApi) diff --git a/internal/bminventory/inventory.go b/internal/bminventory/inventory.go index 65e2130e3d2..89183bd35ab 100644 --- a/internal/bminventory/inventory.go +++ b/internal/bminventory/inventory.go @@ -1323,7 +1323,7 @@ func (b *bareMetalInventory) InstallClusterInternal(ctx context.Context, params } }() - if err = b.generateClusterInstallConfig(asyncCtx, *cluster, clusterInfraenvs); err != nil { + if err = b.generateClusterInstallConfig(asyncCtx, *cluster, clusterInfraenvs, b.ServiceBaseURL); err != nil { return } log.Infof("generated ignition for cluster %s", cluster.ID.String()) @@ -1659,7 +1659,7 @@ func (b *bareMetalInventory) setInstallConfigOverridesUsage(featureUsages string return nil } -func (b *bareMetalInventory) generateClusterInstallConfig(ctx context.Context, cluster common.Cluster, clusterInfraenvs []*common.InfraEnv) error { +func (b *bareMetalInventory) generateClusterInstallConfig(ctx context.Context, cluster common.Cluster, clusterInfraenvs []*common.InfraEnv, serviceBaseURL string) error { log := logutil.FromContext(ctx, b.log) rhRootCa := ignition.RedhatRootCA @@ -1694,7 +1694,7 @@ func (b *bareMetalInventory) generateClusterInstallConfig(ctx context.Context, c installerReleaseImageOverride = *defaultArchImage.URL } - if err := b.generator.GenerateInstallConfig(ctx, cluster, cfg, *releaseImage.URL, installerReleaseImageOverride); err != nil { + if err := b.generator.GenerateInstallConfig(ctx, cluster, cfg, *releaseImage.URL, installerReleaseImageOverride, serviceBaseURL, b.authHandler.AuthType()); err != nil { msg := fmt.Sprintf("failed generating install config for cluster %s", cluster.ID) log.WithError(err).Error(msg) return errors.Wrap(err, msg) @@ -3566,6 +3566,7 @@ func (b *bareMetalInventory) getLogFileForDownload(ctx context.Context, clusterI if err != nil { return "", "", err } + b.log.Debugf("log type to download: %s", logsType) switch logsType { case string(models.LogsTypeHost), string(models.LogsTypeNodeBoot): if hostId == nil { diff --git a/internal/bminventory/inventory_test.go b/internal/bminventory/inventory_test.go index 1bea3558f12..a6e7c51d126 100644 --- a/internal/bminventory/inventory_test.go +++ b/internal/bminventory/inventory_test.go @@ -190,6 +190,7 @@ var ( imageServiceHost = "image-service.example.com:8080" imageServiceBaseURL = fmt.Sprintf("https://%s%s", imageServiceHost, imageServicePath) fakePullSecret = `{\"auths\":{\"cloud.openshift.com\":{\"auth\":\"dG9rZW46dGVzdAo=\",\"email\":\"coyote@acme.com\"}}}"` // #nosec + serviceBaseURL = "https://assisted.example.com:6008" ) func toMac(macStr string) *strfmt.MAC { @@ -297,7 +298,7 @@ func getDefaultClusterCreateParams() *models.ClusterCreateParams { func mockGenerateInstallConfigSuccess(mockGenerator *generator.MockISOInstallConfigGenerator, mockVersions *versions.MockHandler) { if mockGenerator != nil { mockVersions.EXPECT().GetReleaseImage(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(common.TestDefaultConfig.ReleaseImage, nil).Times(1) - mockGenerator.EXPECT().GenerateInstallConfig(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + mockGenerator.EXPECT().GenerateInstallConfig(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), "https://assisted.example.com:6008", getTestAuthHandler().AuthType()).Return(nil).Times(1) } } @@ -5235,7 +5236,7 @@ var _ = Describe("cluster", func() { mockGetInstallConfigSuccess(mockInstallConfigBuilder) mockVersions.EXPECT().GetReleaseImage(gomock.Any(), gomock.Any(), common.ARM64CPUArchitecture, gomock.Any()).Return(armRelease, nil).Times(1) mockVersions.EXPECT().GetReleaseImage(gomock.Any(), gomock.Any(), common.DefaultCPUArchitecture, gomock.Any()).Return(common.TestDefaultConfig.ReleaseImage, nil).Times(1) - mockGenerator.EXPECT().GenerateInstallConfig(gomock.Any(), gomock.Any(), gomock.Any(), *armRelease.URL, *common.TestDefaultConfig.ReleaseImage.URL).Return(nil).Times(1) + mockGenerator.EXPECT().GenerateInstallConfig(gomock.Any(), gomock.Any(), gomock.Any(), *armRelease.URL, *common.TestDefaultConfig.ReleaseImage.URL, "https://assisted.example.com:6008", getTestAuthHandler().AuthType()).Return(nil).Times(1) mockClusterPrepareForInstallationSuccess(mockClusterApi) mockHostPrepareForRefresh(mockHostApi) @@ -15122,6 +15123,7 @@ func createInventory(db *gorm.DB, cfg Config) *bareMetalInventory { mockStaticNetworkConfig, gcConfig, mockProviderRegistry, true) bm.ImageServiceBaseURL = imageServiceBaseURL + bm.ServiceBaseURL = serviceBaseURL return bm } diff --git a/internal/bminventory/inventory_v2_handlers.go b/internal/bminventory/inventory_v2_handlers.go index 0ef4b6cdeae..d78d76fab75 100644 --- a/internal/bminventory/inventory_v2_handlers.go +++ b/internal/bminventory/inventory_v2_handlers.go @@ -386,6 +386,7 @@ func (b *bareMetalInventory) V2DownloadClusterLogs(ctx context.Context, params i log := logutil.FromContext(ctx, b.log) log.Infof("Downloading logs from cluster %s", params.ClusterID) fileName, downloadFileName, err := b.getLogFileForDownload(ctx, ¶ms.ClusterID, params.HostID, swag.StringValue(params.LogsType)) + log.Debugf("file details: logs type=%s, filename=%s, downloadFileName=%s", fileName, downloadFileName, swag.StringValue(params.LogsType)) if err != nil { return common.GenerateErrorResponder(err) } diff --git a/internal/cluster/cluster.go b/internal/cluster/cluster.go index 6121b8806c8..d013dc7d5ba 100644 --- a/internal/cluster/cluster.go +++ b/internal/cluster/cluster.go @@ -1055,7 +1055,12 @@ func (m *Manager) PrepareClusterLogFile(ctx context.Context, c *common.Cluster, if hostObject.Bootstrap { role = string(models.HostRoleBootstrap) } - tarredFilename = fmt.Sprintf("%s_%s_%s.tar.gz", sanitize.Name(c.Name), role, sanitize.Name(hostutil.GetHostnameForMsg(hostObject))) + name := sanitize.Name(hostutil.GetHostnameForMsg(hostObject)) + if strings.Contains(file, "boot_") { + name = fmt.Sprintf("boot_%s", name) + + } + tarredFilename = fmt.Sprintf("%s_%s_%s.tar.gz", sanitize.Name(c.Name), role, name) } } else { tarredFilename = fmt.Sprintf("%s_%s", fileNameSplit[len(fileNameSplit)-2], fileNameSplit[len(fileNameSplit)-1]) diff --git a/internal/ignition/boot-reporter/assisted-boot-reporter.sh b/internal/ignition/boot-reporter/assisted-boot-reporter.sh new file mode 100755 index 00000000000..3025b9043bf --- /dev/null +++ b/internal/ignition/boot-reporter/assisted-boot-reporter.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash + +export LOG_SEND_FREQUENCY_IN_MINUTES=5 +export SERVICE_TIMEOUT_MINUTES=60 + +function log() { + echo "$(date '+%F %T') ${HOSTNAME} $2[$$]: level=$1 msg=\"$3\"" +} + +function log_info() { + log "info" "$1" "$2" +} + +function log_error() { + log "error" "$1" "$2" +} + +function init_variables() { + func_name=${FUNCNAME[0]} + init_failed="false" + + if [ "$ASSISTED_SERVICE_URL" == "" ]; then + init_failed="true" + log_error "${func_name}" "ASSISTED_SERVICE_URL is empty." + elif [ "${ASSISTED_SERVICE_URL: -1}" == "/" ]; then + export ASSISTED_SERVICE_URL="${ASSISTED_SERVICE_URL::-1}" + fi + + if [ "$CLUSTER_ID" == "" ]; then + init_failed="true" + log_error "${func_name}" "CLUSTER_ID is empty." + fi + + if [ "$INFRA_ENV_ID" == "" ]; then + init_failed="true" + log_error "${func_name}" "INFRA_ENV_ID is empty." + fi + + if [ "$HOST_ID" == "" ]; then + init_failed="true" + log_error "${func_name}" "HOST_ID is empty." + fi + + if [ "$init_failed" == "true" ]; then + log_error "${func_name}" "Failed to initialize variables. Exiting." + exit 1 + fi +} + +function collect_and_upload_logs() { + func_name=${FUNCNAME[0]} + + log_info "${func_name}" "Collecting logs." + logs_dir_name=boot_logs_$HOST_ID + logs_path=/tmp/$logs_dir_name + + rm -rf $logs_path + mkdir -p $logs_path + + journalctl > "$logs_path"/journalctl.log + log_info "${func_name}" "Copying journalctl to $logs_path/" + ip a > $logs_path/ip_a.log + log_info "${func_name}" "Capturing the output of 'ip a' to $logs_path/" + cp /etc/resolv.conf $logs_path + log_info "${func_name}" "Copying /etc/resolv.conf to $logs_path/" + + pushd /tmp + log_info "${func_name}" "Compressing logs to $logs_dir_name.tar.gz" + tar -czvf "$logs_dir_name".tar.gz "$logs_dir_name" + popd + + log_info "${func_name}" "Uploading logs." + + curl -X POST -H "X-Secret-Key: ${PULL_SECRET_TOKEN}" \ + -F upfile=@$logs_path.tar.gz \ + "$ASSISTED_SERVICE_URL/api/assisted-install/v2/clusters/$CLUSTER_ID/logs?logs_type=node-boot&infra_env_id=$INFRA_ENV_ID&host_id=$HOST_ID" + + if [ $? -eq 0 ]; then + log_info "${func_name}" "Successfully uploaded logs." + else + log_error "${func_name}" "Failed to upload logs." + fi +} + +function main() { + func_name=${FUNCNAME[0]} + count=$((SERVICE_TIMEOUT_MINUTES/LOG_SEND_FREQUENCY_IN_MINUTES)) + + for i in $(seq $count) + do + log_info "${func_name}" "Upload logs attempt ${i}/${count}" + collect_and_upload_logs + if [ "$i" != "$count" ]; then # don't sleep at the last iteration. + log_info "${func_name}" "Sleeping for ${LOG_SEND_ITERATION_MINUTES} minutes until the next attempt." + sleep $((LOG_SEND_FREQUENCY_IN_MINUTES*60)) + fi + done +} + +log_info assisted-boot-reporter "assisted-boot-reporter start" +init_variables +main +log_info assisted-boot-reporter "assisted-boot-reporter end" +exit 0 diff --git a/internal/ignition/dummy.go b/internal/ignition/dummy.go index d09aebe64e3..d605db5a8ab 100644 --- a/internal/ignition/dummy.go +++ b/internal/ignition/dummy.go @@ -8,29 +8,35 @@ import ( "github.com/openshift/assisted-service/internal/common" "github.com/openshift/assisted-service/internal/host/hostutil" "github.com/openshift/assisted-service/models" + "github.com/openshift/assisted-service/pkg/auth" "github.com/openshift/assisted-service/pkg/s3wrapper" "github.com/sirupsen/logrus" + "gorm.io/gorm" ) type dummyGenerator struct { - log logrus.FieldLogger - workDir string - cluster *common.Cluster - s3Client s3wrapper.API + log logrus.FieldLogger + db *gorm.DB + serviceBaseURL string + workDir string + cluster *common.Cluster + s3Client s3wrapper.API } // NewDummyGenerator returns a Generator that creates the expected files but with nonsense content -func NewDummyGenerator(workDir string, cluster *common.Cluster, s3Client s3wrapper.API, log logrus.FieldLogger) Generator { +func NewDummyGenerator(db *gorm.DB, serviceBaseURL string, workDir string, cluster *common.Cluster, s3Client s3wrapper.API, log logrus.FieldLogger) Generator { return &dummyGenerator{ - workDir: workDir, - log: log, - cluster: cluster, - s3Client: s3Client, + workDir: workDir, + log: log, + db: db, + serviceBaseURL: serviceBaseURL, + cluster: cluster, + s3Client: s3Client, } } // Generate creates the expected ignition and related files but with nonsense content -func (g *dummyGenerator) Generate(_ context.Context, installConfig []byte, platformType models.PlatformType) error { +func (g *dummyGenerator) Generate(_ context.Context, installConfig []byte, platformType models.PlatformType, serviceBaseURL string, authType auth.AuthType) error { toUpload := fileNames[:] for _, host := range g.cluster.Hosts { toUpload = append(toUpload, hostutil.IgnitionFileName(host)) diff --git a/internal/ignition/ignition.go b/internal/ignition/ignition.go index a80a6fcd378..0478b92fce3 100644 --- a/internal/ignition/ignition.go +++ b/internal/ignition/ignition.go @@ -51,6 +51,7 @@ import ( "github.com/vincent-petithory/dataurl" "golang.org/x/sync/errgroup" "gopkg.in/yaml.v2" + "gorm.io/gorm" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sjson "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/client-go/kubernetes/scheme" @@ -191,6 +192,28 @@ const highlyAvailableInfrastructureTopologyPatch = `--- const tempNMConnectionsDir = "/etc/assisted/network" +const bootReporterPath = "/assisted-boot-reporter.sh" + +var assistedBootReporterunitTemplate = `[Unit] +Description=Collect and upload host boot logs to assisted-service +Wants=network-online.target +After=network-online.target +DefaultDependencies=no +[Service] +Environment=ASSISTED_SERVICE_URL=%s +Environment=PULL_SECRET_TOKEN=%s +Environment=CLUSTER_ID=%s +Environment=INFRA_ENV_ID=%s +Environment=HOST_ID=%s +User=root +Type=oneshot +ExecStart=/bin/bash /usr/local/bin/assisted-boot-reporter.sh +PrivateTmp=true +RemainAfterExit=no +[Install] +WantedBy=multi-user.target +` + var fileNames = [...]string{ "bootstrap.ign", masterIgn, @@ -203,7 +226,7 @@ var fileNames = [...]string{ // Generator can generate ignition files and upload them to an S3-like service type Generator interface { - Generate(ctx context.Context, installConfig []byte, platformType models.PlatformType) error + Generate(ctx context.Context, installConfig []byte, platformType models.PlatformType, serviceBaseURL string, authType auth.AuthType) error UploadToS3(ctx context.Context) error UpdateEtcHosts(string) error } @@ -218,6 +241,8 @@ type IgnitionBuilder interface { type installerGenerator struct { log logrus.FieldLogger + db *gorm.DB + serviceBaseURL string workDir string cluster *common.Cluster releaseImage string @@ -276,12 +301,14 @@ func NewBuilder(log logrus.FieldLogger, staticNetworkConfig staticnetworkconfig. } // NewGenerator returns a generator that can generate ignition files -func NewGenerator(workDir string, installerDir string, cluster *common.Cluster, releaseImage string, releaseImageMirror string, - serviceCACert, installInvoker string, s3Client s3wrapper.API, log logrus.FieldLogger, operatorsApi operators.API, +func NewGenerator(db *gorm.DB, serviceBaseURL string, workDir string, installerDir string, cluster *common.Cluster, releaseImage string, releaseImageMirror string, + serviceCACert string, installInvoker string, s3Client s3wrapper.API, log logrus.FieldLogger, operatorsApi operators.API, providerRegistry registry.ProviderRegistry, installerReleaseImageOverride, clusterTLSCertOverrideDir string) Generator { return &installerGenerator{ cluster: cluster, log: log, + db: db, + serviceBaseURL: serviceBaseURL, releaseImage: releaseImage, releaseImageMirror: releaseImageMirror, workDir: workDir, @@ -304,7 +331,7 @@ func (g *installerGenerator) UploadToS3(ctx context.Context) error { } // Generate generates ignition files and applies modifications. -func (g *installerGenerator) Generate(ctx context.Context, installConfig []byte, platformType models.PlatformType) error { +func (g *installerGenerator) Generate(ctx context.Context, installConfig []byte, platformType models.PlatformType, serviceBaseURL string, authType auth.AuthType) error { var icspFile string var err error log := logutil.FromContext(ctx, g.log) @@ -440,7 +467,7 @@ func (g *installerGenerator) Generate(ctx context.Context, installConfig []byte, return err } - err = g.createHostIgnitions() + err = g.createHostIgnitions(bootReporterPath, serviceBaseURL, authType) if err != nil { log.Error(err) return err @@ -1257,15 +1284,54 @@ func HasCACertInIgnition(contents string) bool { return len(config.Ignition.Security.TLS.CertificateAuthorities) > 0 } -func writeHostFiles(hosts []*models.Host, baseFile string, workDir string) error { - g := new(errgroup.Group) +func getBootReporterFileContent(path string) (string, error) { + bootReporterContent, err := os.ReadFile(path) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(bootReporterContent), nil +} + +func (g *installerGenerator) getHostsInfraEnvs(hosts []*models.Host) (map[string]*common.InfraEnv, error) { + var infraEnvs = map[string]*common.InfraEnv{} for i := range hosts { host := hosts[i] - g.Go(func() error { + if _, ok := infraEnvs[host.InfraEnvID.String()]; !ok { + infraEnv, err := common.GetInfraEnvFromDB(g.db, host.InfraEnvID) + if err != nil { + return nil, err + } + infraEnvs[host.InfraEnvID.String()] = infraEnv + } + } + return infraEnvs, nil +} + +func (g *installerGenerator) writeHostFiles(hosts []*models.Host, baseFile string, workDir string, bootReporterPath string, serviceBaseURL string, authType auth.AuthType) error { + errGroup := new(errgroup.Group) + bootReporter, err := getBootReporterFileContent(bootReporterPath) + if err != nil { + return errors.Wrap(err, "failed to read the contents of assisted-boot-reporter.sh") + } + infraEnvs, err := g.getHostsInfraEnvs(hosts) + if err != nil { + return err + } + + for i := range hosts { + host := hosts[i] + errGroup.Go(func() error { config, err := parseIgnitionFile(filepath.Join(workDir, baseFile)) if err != nil { return err } + infraEnv := infraEnvs[host.InfraEnvID.String()] + pullSecretToken, err := clusterPkg.AgentToken(infraEnv, authType) + if err != nil { + return err + } + contents := fmt.Sprintf(assistedBootReporterunitTemplate, strings.TrimSpace(serviceBaseURL), pullSecretToken, host.ClusterID, host.InfraEnvID, host.ID) + setUnitInIgnition(config, contents, "assisted-boot-reporter.service", true) hostname, err := hostutil.GetCurrentHostName(host) if err != nil { @@ -1274,6 +1340,8 @@ func writeHostFiles(hosts []*models.Host, baseFile string, workDir string) error setFileInIgnition(config, "/etc/hostname", fmt.Sprintf("data:,%s", hostname), false, 420, true) + setFileInIgnition(config, "/usr/local/bin/assisted-boot-reporter.sh", fmt.Sprintf("data:text/plain;charset=utf-8;base64,%s", bootReporter), false, 0700, true) + configBytes, err := json.Marshal(config) if err != nil { return err @@ -1296,19 +1364,19 @@ func writeHostFiles(hosts []*models.Host, baseFile string, workDir string) error }) } - return g.Wait() + return errGroup.Wait() } // createHostIgnitions builds an ignition file for each host in the cluster based on the generated .ign file -func (g *installerGenerator) createHostIgnitions() error { +func (g *installerGenerator) createHostIgnitions(bootReporterPath string, serviceBaseURL string, authType auth.AuthType) error { masters, workers := sortHosts(g.cluster.Hosts) - err := writeHostFiles(masters, masterIgn, g.workDir) + err := g.writeHostFiles(masters, masterIgn, g.workDir, bootReporterPath, serviceBaseURL, authType) if err != nil { return errors.Wrapf(err, "error writing master host ignition files") } - err = writeHostFiles(workers, workerIgn, g.workDir) + err = g.writeHostFiles(workers, workerIgn, g.workDir, bootReporterPath, serviceBaseURL, authType) if err != nil { return errors.Wrapf(err, "error writing worker host ignition files") } diff --git a/internal/ignition/ignition_test.go b/internal/ignition/ignition_test.go index 14af8fae1c6..b505165d1e7 100644 --- a/internal/ignition/ignition_test.go +++ b/internal/ignition/ignition_test.go @@ -36,6 +36,7 @@ import ( "github.com/sirupsen/logrus" "github.com/vincent-petithory/dataurl" "gopkg.in/yaml.v2" + "gorm.io/gorm" ) var ( @@ -49,6 +50,19 @@ var ( ctrl *gomock.Controller ) +func createInfraEnv(db *gorm.DB, id strfmt.UUID, clusterID strfmt.UUID) *common.InfraEnv { + infraEnv := &common.InfraEnv{ + PullSecret: "{\"auths\":{\"cloud.openshift.com\":{\"auth\":\"dG9rZW46dGVzdAo=\",\"email\":\"coyote@acme.com\"}}}", + InfraEnv: models.InfraEnv{ + ID: &id, + ClusterID: clusterID, + PullSecretSet: true, + }, + } + Expect(db.Create(infraEnv).Error).ToNot(HaveOccurred()) + return infraEnv +} + var _ = BeforeEach(func() { // setup temp workdir var err error @@ -108,6 +122,8 @@ var _ = Describe("Bootstrap Ignition Update", func() { var ( err error examplePath string + db *gorm.DB + dbName string bmh *bmh_v1alpha1.BareMetalHost config *config_32_types.Config mockS3Client *s3wrapper.MockAPI @@ -127,7 +143,8 @@ var _ = Describe("Bootstrap Ignition Update", func() { Role: models.HostRoleMaster, }, } - g := NewGenerator(workDir, installerCacheDir, cluster, "", "", "", "", mockS3Client, log, + db, dbName = common.PrepareTestDB() + g := NewGenerator(db, "", workDir, installerCacheDir, cluster, "", "", "", "", mockS3Client, log, mockOperatorManager, mockProviderRegistry, "", "").(*installerGenerator) err = g.updateBootstrap(context.Background(), examplePath) @@ -157,6 +174,10 @@ var _ = Describe("Bootstrap Ignition Update", func() { Expect(foundNMConfig).To(BeTrue(), "file /etc/NetworkManager/conf.d/99-kni.conf not present in bootstrap.ign") }) + AfterEach(func() { + common.DeleteTestDB(db, dbName) + }) + Context("Identify host role", func() { var hosts []*models.Host @@ -253,6 +274,8 @@ SV4bRR9i0uf+xQ/oYRvugQ25Q7EahO5hJIWRf4aULbk36Zpw3++v2KFnF26zqwB6 masterPath string workerPath string caCertPath string + dbName string + db *gorm.DB ) BeforeEach(func() { @@ -266,11 +289,16 @@ SV4bRR9i0uf+xQ/oYRvugQ25Q7EahO5hJIWRf4aULbk36Zpw3++v2KFnF26zqwB6 caCertPath = filepath.Join(workDir, "service-ca-cert.crt") err = os.WriteFile(caCertPath, []byte(caCert), 0600) Expect(err).NotTo(HaveOccurred()) + db, dbName = common.PrepareTestDB() + }) + + AfterEach(func() { + common.DeleteTestDB(db, dbName) }) Describe("update ignitions", func() { It("with ca cert file", func() { - g := NewGenerator(workDir, installerCacheDir, cluster, "", "", caCertPath, "", nil, log, + g := NewGenerator(db, "", workDir, installerCacheDir, cluster, "", "", caCertPath, "", nil, log, mockOperatorManager, mockProviderRegistry, "", "").(*installerGenerator) err := g.updateIgnitions() @@ -293,7 +321,7 @@ SV4bRR9i0uf+xQ/oYRvugQ25Q7EahO5hJIWRf4aULbk36Zpw3++v2KFnF26zqwB6 Expect(file.Path).To(Equal(common.HostCACertPath)) }) It("with no ca cert file", func() { - g := NewGenerator(workDir, installerCacheDir, cluster, "", "", "", "", nil, log, + g := NewGenerator(db, "", workDir, installerCacheDir, cluster, "", "", "", "", nil, log, mockOperatorManager, mockProviderRegistry, "", "").(*installerGenerator) err := g.updateIgnitions() @@ -312,7 +340,7 @@ SV4bRR9i0uf+xQ/oYRvugQ25Q7EahO5hJIWRf4aULbk36Zpw3++v2KFnF26zqwB6 Expect(workerConfig.Storage.Files).To(HaveLen(0)) }) It("with service ips", func() { - g := NewGenerator(workDir, installerCacheDir, cluster, "", "", "", "", nil, log, + g := NewGenerator(db, "", workDir, installerCacheDir, cluster, "", "", "", "", nil, log, mockOperatorManager, mockProviderRegistry, "", "").(*installerGenerator) err := g.UpdateEtcHosts("10.10.10.1,10.10.10.2") @@ -335,7 +363,7 @@ SV4bRR9i0uf+xQ/oYRvugQ25Q7EahO5hJIWRf4aULbk36Zpw3++v2KFnF26zqwB6 Expect(file.Path).To(Equal("/etc/hosts")) }) It("with no service ips", func() { - g := NewGenerator(workDir, installerCacheDir, cluster, "", "", "", "", nil, log, + g := NewGenerator(db, "", workDir, installerCacheDir, cluster, "", "", "", "", nil, log, mockOperatorManager, mockProviderRegistry, "", "").(*installerGenerator) err := g.UpdateEtcHosts("") @@ -365,7 +393,7 @@ SV4bRR9i0uf+xQ/oYRvugQ25Q7EahO5hJIWRf4aULbk36Zpw3++v2KFnF26zqwB6 }) Context("DHCP generation", func() { It("Definitions only", func() { - g := NewGenerator(workDir, installerCacheDir, cluster, "", "", "", "", nil, log, + g := NewGenerator(db, "", workDir, installerCacheDir, cluster, "", "", "", "", nil, log, mockOperatorManager, mockProviderRegistry, "", "").(*installerGenerator) g.encodedDhcpFileContents = "data:,abc" @@ -384,7 +412,7 @@ SV4bRR9i0uf+xQ/oYRvugQ25Q7EahO5hJIWRf4aULbk36Zpw3++v2KFnF26zqwB6 }) }) It("Definitions+leases", func() { - g := NewGenerator(workDir, installerCacheDir, cluster, "", "", "", "", nil, log, + g := NewGenerator(db, "", workDir, installerCacheDir, cluster, "", "", "", "", nil, log, mockOperatorManager, mockProviderRegistry, "", "").(*installerGenerator) g.encodedDhcpFileContents = "data:,abc" @@ -471,6 +499,11 @@ var _ = Describe("createHostIgnitions", func() { } }` + var ( + dbName string + db *gorm.DB + ) + BeforeEach(func() { masterPath := filepath.Join(workDir, "master.ign") err := os.WriteFile(masterPath, []byte(masterIgn), 0600) @@ -479,10 +512,16 @@ var _ = Describe("createHostIgnitions", func() { workerPath := filepath.Join(workDir, "worker.ign") err = os.WriteFile(workerPath, []byte(workerIgn), 0600) Expect(err).NotTo(HaveOccurred()) + db, dbName = common.PrepareTestDB() + + }) + + AfterEach(func() { + common.DeleteTestDB(db, dbName) }) Context("with multiple hosts with a hostname", func() { - It("adds the hostname file", func() { + It("adds the hostname and boot-reporter files", func() { cluster.Hosts = []*models.Host{ { RequestedHostname: "master0.example.com", @@ -503,15 +542,18 @@ var _ = Describe("createHostIgnitions", func() { } // create an ID for each host + infraEnvId := strfmt.UUID(uuid.New().String()) + createInfraEnv(db, infraEnvId, *cluster.ID) for _, host := range cluster.Hosts { id := strfmt.UUID(uuid.New().String()) host.ID = &id + host.InfraEnvID = infraEnvId } - g := NewGenerator(workDir, installerCacheDir, cluster, "", "", "", "", nil, log, + g := NewGenerator(db, "", workDir, installerCacheDir, cluster, "", "", "", "", nil, log, mockOperatorManager, mockProviderRegistry, "", "").(*installerGenerator) - err := g.createHostIgnitions() + err := g.createHostIgnitions("./boot-reporter"+bootReporterPath, "http://www.example.com:6008", auth.TypeRHSSO) Expect(err).NotTo(HaveOccurred()) for _, host := range cluster.Hosts { @@ -541,23 +583,43 @@ var _ = Describe("createHostIgnitions", func() { Expect(*f.FileEmbedded1.Contents.Source).To(Equal(fmt.Sprintf("data:,%s", host.RequestedHostname))) Expect(*f.FileEmbedded1.Mode).To(Equal(420)) Expect(*f.Node.Overwrite).To(Equal(true)) + + By("Validating the boot-reporter file was added") + var bootReporterFile *config_32_types.File + base64Content, _ := getBootReporterFileContent("./boot-reporter" + bootReporterPath) + + for fileidx, file := range config.Storage.Files { + if file.Node.Path == "/usr/local/bin/assisted-boot-reporter.sh" { + bootReporterFile = &config.Storage.Files[fileidx] + break + } + } + Expect(bootReporterFile).NotTo(BeNil()) + Expect(*bootReporterFile.Node.User.Name).To(Equal("root")) + Expect(*bootReporterFile.FileEmbedded1.Contents.Source).To(Equal(fmt.Sprintf("data:text/plain;charset=utf-8;base64,%s", base64Content))) + Expect(*bootReporterFile.FileEmbedded1.Mode).To(Equal(0700)) + Expect(*bootReporterFile.Node.Overwrite).To(Equal(true)) + } }) }) It("applies overrides correctly", func() { hostID := strfmt.UUID(uuid.New().String()) + infraEnvId := strfmt.UUID(uuid.New().String()) + createInfraEnv(db, infraEnvId, *cluster.ID) cluster.Hosts = []*models.Host{{ ID: &hostID, + InfraEnvID: infraEnvId, RequestedHostname: "master0.example.com", Role: models.HostRoleMaster, IgnitionConfigOverrides: `{"ignition": {"version": "3.2.0"}, "storage": {"files": [{"path": "/tmp/example", "contents": {"source": "data:text/plain;base64,aGVscGltdHJhcHBlZGluYXN3YWdnZXJzcGVj"}}]}}`, }} - g := NewGenerator(workDir, installerCacheDir, cluster, "", "", "", "", nil, log, + g := NewGenerator(db, "", workDir, installerCacheDir, cluster, "", "", "", "", nil, log, mockOperatorManager, mockProviderRegistry, "", "").(*installerGenerator) - err := g.createHostIgnitions() + err := g.createHostIgnitions("./boot-reporter"+bootReporterPath, "http://www.example.com:6008", auth.TypeNone) Expect(err).NotTo(HaveOccurred()) ignBytes, err := os.ReadFile(filepath.Join(workDir, fmt.Sprintf("%s-%s.ign", models.HostRoleMaster, hostID))) @@ -1592,7 +1654,11 @@ var _ = Describe("FormatSecondDayWorkerIgnitionFile", func() { }) var _ = Describe("Import Cluster TLS Certs for ephemeral installer", func() { - var certDir string + var ( + certDir string + dbName string + db *gorm.DB + ) certFiles := []string{"test-cert.crt", "test-cert.key"} @@ -1605,10 +1671,16 @@ var _ = Describe("Import Cluster TLS Certs for ephemeral installer", func() { err = os.WriteFile(filepath.Join(certDir, cf), []byte(cf), 0600) Expect(err).NotTo(HaveOccurred()) } + Expect(err).NotTo(HaveOccurred()) + db, dbName = common.PrepareTestDB() + }) + + AfterEach(func() { + common.DeleteTestDB(db, dbName) }) It("copies the tls cert files", func() { - g := NewGenerator(workDir, installerCacheDir, cluster, "", "", "", "", nil, log, + g := NewGenerator(db, "", workDir, installerCacheDir, cluster, "", "", "", "", nil, log, mockOperatorManager, mockProviderRegistry, "", certDir).(*installerGenerator) err := g.importClusterTLSCerts(context.Background()) diff --git a/internal/ignition/mock_ignition.go b/internal/ignition/mock_ignition.go index 26576110631..0a58aa6a8a4 100644 --- a/internal/ignition/mock_ignition.go +++ b/internal/ignition/mock_ignition.go @@ -38,17 +38,17 @@ func (m *MockGenerator) EXPECT() *MockGeneratorMockRecorder { } // Generate mocks base method. -func (m *MockGenerator) Generate(ctx context.Context, installConfig []byte, platformType models.PlatformType) error { +func (m *MockGenerator) Generate(ctx context.Context, installConfig []byte, platformType models.PlatformType, serviceBaseURL string, authType auth.AuthType) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Generate", ctx, installConfig, platformType) + ret := m.ctrl.Call(m, "Generate", ctx, installConfig, platformType, serviceBaseURL, authType) ret0, _ := ret[0].(error) return ret0 } // Generate indicates an expected call of Generate. -func (mr *MockGeneratorMockRecorder) Generate(ctx, installConfig, platformType interface{}) *gomock.Call { +func (mr *MockGeneratorMockRecorder) Generate(ctx, installConfig, platformType, serviceBaseURL, authType interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Generate", reflect.TypeOf((*MockGenerator)(nil).Generate), ctx, installConfig, platformType) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Generate", reflect.TypeOf((*MockGenerator)(nil).Generate), ctx, installConfig, platformType, serviceBaseURL, authType) } // UpdateEtcHosts mocks base method. diff --git a/internal/ignition/templates/assisted-boot-reporter.service b/internal/ignition/templates/assisted-boot-reporter.service new file mode 100644 index 00000000000..92b2cec01ab --- /dev/null +++ b/internal/ignition/templates/assisted-boot-reporter.service @@ -0,0 +1,17 @@ +[Unit] +Description=Collect and upload host boot logs to assisted-service +Wants=network-online.target +After=network-online.target +DefaultDependencies=no +[Service] +Environment=PULL_SECRET_TOKEN={{.PullSecretToken}} +Environment=CLUSTER_ID={{.clusterId}} +Environment=INFRA_ENV_ID={{.infraEnvId}} +Environment=HOST_ID={{.hostId}} +User=root +Type=oneshot +ExecStart=/bin/bash /usr/local/bin/assisted-boot-reporter.sh +PrivateTmp=true +RemainAfterExit=no +[Install] +WantedBy=multi-user.target diff --git a/internal/ignition/templates/node.ign b/internal/ignition/templates/node.ign index e348c6ca763..376a718a67f 100644 --- a/internal/ignition/templates/node.ign +++ b/internal/ignition/templates/node.ign @@ -15,4 +15,4 @@ } }{{end}} } -} +} \ No newline at end of file diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index 0d5495b97ab..1245aa7fbc1 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -11,13 +11,15 @@ import ( "github.com/openshift/assisted-service/internal/operators" "github.com/openshift/assisted-service/internal/provider/registry" "github.com/openshift/assisted-service/models" + "github.com/openshift/assisted-service/pkg/auth" logutil "github.com/openshift/assisted-service/pkg/log" "github.com/openshift/assisted-service/pkg/s3wrapper" "github.com/sirupsen/logrus" + "gorm.io/gorm" ) type InstallConfigGenerator interface { - GenerateInstallConfig(ctx context.Context, cluster common.Cluster, cfg []byte, releaseImage, installerReleaseImageOverride string) error + GenerateInstallConfig(ctx context.Context, cluster common.Cluster, cfg []byte, releaseImage, installerReleaseImageOverride, serviceBaseURL string, authType auth.AuthType) error } //go:generate mockgen --build_flags=--mod=mod -package generator -destination mock_install_config.go . ISOInstallConfigGenerator @@ -36,6 +38,7 @@ type Config struct { type installGenerator struct { Config log logrus.FieldLogger + db *gorm.DB s3Client s3wrapper.API operatorsApi operators.API workDir string @@ -43,11 +46,12 @@ type installGenerator struct { clusterTLSCertOverrideDir string } -func New(log logrus.FieldLogger, s3Client s3wrapper.API, cfg Config, workDir string, +func New(log logrus.FieldLogger, db *gorm.DB, s3Client s3wrapper.API, cfg Config, workDir string, operatorsApi operators.API, providerRegistry registry.ProviderRegistry, clusterTLSCertOverrideDir string) *installGenerator { return &installGenerator{ Config: cfg, log: log, + db: db, s3Client: s3Client, operatorsApi: operatorsApi, workDir: filepath.Join(workDir, "install-config-generate"), @@ -57,7 +61,7 @@ func New(log logrus.FieldLogger, s3Client s3wrapper.API, cfg Config, workDir str } // GenerateInstallConfig creates install config and ignition files -func (k *installGenerator) GenerateInstallConfig(ctx context.Context, cluster common.Cluster, cfg []byte, releaseImage, installerReleaseImageOverride string) error { +func (k *installGenerator) GenerateInstallConfig(ctx context.Context, cluster common.Cluster, cfg []byte, releaseImage, installerReleaseImageOverride, serviceBaseURL string, authType auth.AuthType) error { log := logutil.FromContext(ctx, k.log) err := os.MkdirAll(k.workDir, 0o755) if err != nil { @@ -94,12 +98,12 @@ func (k *installGenerator) GenerateInstallConfig(ctx context.Context, cluster co // runs openshift-install to generate ignition files, then modifies them as necessary var generator ignition.Generator if k.Config.DummyIgnition { - generator = ignition.NewDummyGenerator(clusterWorkDir, &cluster, k.s3Client, log) + generator = ignition.NewDummyGenerator(k.db, serviceBaseURL, clusterWorkDir, &cluster, k.s3Client, log) } else { - generator = ignition.NewGenerator(clusterWorkDir, installerCacheDir, &cluster, releaseImage, k.Config.ReleaseImageMirror, + generator = ignition.NewGenerator(k.db, serviceBaseURL, clusterWorkDir, installerCacheDir, &cluster, releaseImage, k.Config.ReleaseImageMirror, k.Config.ServiceCACertPath, k.Config.InstallInvoker, k.s3Client, log, k.operatorsApi, k.providerRegistry, installerReleaseImageOverride, k.clusterTLSCertOverrideDir) } - err = generator.Generate(ctx, cfg, k.getClusterPlatformType(cluster)) + err = generator.Generate(ctx, cfg, k.getClusterPlatformType(cluster), serviceBaseURL, authType) if err != nil { return err } diff --git a/pkg/generator/mock_install_config.go b/pkg/generator/mock_install_config.go index d19ed28ec7c..8c9c97868cc 100644 --- a/pkg/generator/mock_install_config.go +++ b/pkg/generator/mock_install_config.go @@ -10,6 +10,7 @@ import ( gomock "github.com/golang/mock/gomock" common "github.com/openshift/assisted-service/internal/common" + auth "github.com/openshift/assisted-service/pkg/auth" ) // MockISOInstallConfigGenerator is a mock of ISOInstallConfigGenerator interface. @@ -36,15 +37,15 @@ func (m *MockISOInstallConfigGenerator) EXPECT() *MockISOInstallConfigGeneratorM } // GenerateInstallConfig mocks base method. -func (m *MockISOInstallConfigGenerator) GenerateInstallConfig(arg0 context.Context, arg1 common.Cluster, arg2 []byte, arg3, arg4 string) error { +func (m *MockISOInstallConfigGenerator) GenerateInstallConfig(arg0 context.Context, arg1 common.Cluster, arg2 []byte, arg3, arg4, arg5 string, arg6 auth.AuthType) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GenerateInstallConfig", arg0, arg1, arg2, arg3, arg4) + ret := m.ctrl.Call(m, "GenerateInstallConfig", arg0, arg1, arg2, arg3, arg4, arg5, arg6) ret0, _ := ret[0].(error) return ret0 } // GenerateInstallConfig indicates an expected call of GenerateInstallConfig. -func (mr *MockISOInstallConfigGeneratorMockRecorder) GenerateInstallConfig(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { +func (mr *MockISOInstallConfigGeneratorMockRecorder) GenerateInstallConfig(arg0, arg1, arg2, arg3, arg4, arg5, arg6 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateInstallConfig", reflect.TypeOf((*MockISOInstallConfigGenerator)(nil).GenerateInstallConfig), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateInstallConfig", reflect.TypeOf((*MockISOInstallConfigGenerator)(nil).GenerateInstallConfig), arg0, arg1, arg2, arg3, arg4, arg5, arg6) }