MGMT-13081: CI to use CentOS8

assisted service is expected to build and run on CentOS8 / RHEL8.
This PR is aligning CI to use that instead of CentOS9.

Additionally, some unit tests were fixed to be less flaky when
working with slower databases. Some tests were besing on object
ordering which was never ensured and failed
those tests from time to time.

ci-images/Dockerfile.base

@@ -1,7 +1,7 @@
-FROM quay.io/centos/centos:stream9
+FROM quay.io/centos/centos:stream8
 
 RUN dnf install -y 'dnf-command(config-manager)' && \
-    dnf config-manager --set-enabled crb && \
+    dnf config-manager --set-enabled powertools && \
     dnf install -y git unzip make gcc which nmstate-devel
 
 # Git checks if the user that owns the files on the filesystem match the
@@ -10,6 +10,7 @@ RUN dnf install -y 'dnf-command(config-manager)' && \
 RUN git config --system --add safe.directory '*'
 
 COPY --from=registry.ci.openshift.org/openshift/release:golang-1.18 /usr/local/go /usr/local/go
+COPY --from=registry.ci.openshift.org/openshift/release:golang-1.18 /usr/bin/gotestsum /usr/bin/make /usr/bin/
 
 ENV GOPATH=/go
 ENV GOROOT=/usr/local/go

ci-images/Dockerfile.lint

@@ -1,3 +1,5 @@
 FROM base
 
 COPY --from=quay.io/app-sre/golangci-lint:v1.46.0 /usr/bin/golangci-lint /usr/bin/golangci-lint
+
+RUN dnf install -y diffutils

ci-images/Dockerfile.subsystem

@@ -3,5 +3,7 @@ FROM base
 COPY --from=quay.io/openshift/origin-cli:latest /usr/bin/oc /usr/bin/kubectl /usr/bin/
 COPY --from=k8s.gcr.io/kustomize/kustomize:v4.3.0 /app/kustomize /usr/bin/
 
+RUN dnf install -y openssl
+
 RUN make build-openshift-ci-test-bin
 RUN go mod vendor

internal/cluster/cluster.go

@@ -1311,13 +1311,22 @@ func (m Manager) DeregisterInactiveCluster(ctx context.Context, maxDeregisterPer
 
 	var clusters []*common.Cluster
 
+	fmt.Println("XXXXX: before db.")
+
 	if err := m.db.Limit(maxDeregisterPerInterval).Where("updated_at < ?", inactiveSince).Find(&clusters).Error; err != nil {
+		fmt.Printf("XXXXX: db err: %s", err.Error())
 		return err
 	}
+	logrus.Println("XXXXX: before loop.")
+	logrus.Printf("XXXXX: clusters len is: %d", len(clusters))
 	for _, c := range clusters {
+		fmt.Printf("XXXXX: about to send event! %s", c.ID.String())
 		eventgen.SendAfterInactivityClusterDeregisteredEvent(ctx, m.eventsHandler, *c.ID)
+		fmt.Printf("XXXXX: Cluster %s is deregistered due to inactivity since %s", c.ID, c.UpdatedAt)
 		log.Infof("Cluster %s is deregistered due to inactivity since %s", c.ID, c.UpdatedAt)
 		if err := m.DeregisterCluster(ctx, c); err != nil {
+			logrus.Printf("failed to deregister inactive cluster %s: %s ", c.ID, err.Error())
+
 			log.WithError(err).Errorf("failed to deregister inactive cluster %s ", c.ID)
 			continue
 		}

internal/cluster/cluster_test.go

@@ -1545,7 +1545,8 @@ var _ = Describe("CancelInstallation", func() {
 			events, err := eventsHandler.V2GetEvents(ctx, c.ID, nil, nil)
 			Expect(err).ShouldNot(HaveOccurred())
 			Expect(len(events)).ShouldNot(Equal(0))
-			cancelEvent := events[len(events)-1]
+			cancelEvent := FindEventByName(events, eventgen.ClusterInstallationCanceledEventName)
+			Expect(cancelEvent).NotTo(BeNil())
 			Expect(*cancelEvent.Severity).Should(Equal(models.EventSeverityInfo))
 			Expect(*cancelEvent.Message).Should(Equal("Canceled cluster installation"))
 		})
@@ -1557,7 +1558,8 @@ var _ = Describe("CancelInstallation", func() {
 			events, err := eventsHandler.V2GetEvents(ctx, c.ID, nil, nil)
 			Expect(err).ShouldNot(HaveOccurred())
 			Expect(len(events)).ShouldNot(Equal(0))
-			cancelEvent := events[len(events)-1]
+			cancelEvent := FindEventByName(events, eventgen.ClusterInstallationCanceledEventName)
+			Expect(cancelEvent).NotTo(BeNil())
 			Expect(*cancelEvent.Severity).Should(Equal(models.EventSeverityInfo))
 			Expect(*cancelEvent.Message).Should(Equal("Canceled cluster installation"))
 		})
@@ -1575,7 +1577,8 @@ var _ = Describe("CancelInstallation", func() {
 			events, err := eventsHandler.V2GetEvents(ctx, c.ID, nil, nil)
 			Expect(err).ShouldNot(HaveOccurred())
 			Expect(len(events)).ShouldNot(Equal(0))
-			cancelEvent := events[len(events)-1]
+			cancelEvent := FindEventByName(events, eventgen.CancelInstallationFailedEventName)
+			Expect(cancelEvent).NotTo(BeNil())
 			Expect(*cancelEvent.Severity).Should(Equal(models.EventSeverityError))
 		})
 	})
@@ -1620,7 +1623,8 @@ var _ = Describe("ResetCluster", func() {
 		events, err := eventsHandler.V2GetEvents(ctx, c.ID, nil, nil)
 		Expect(err).ShouldNot(HaveOccurred())
 		Expect(len(events)).ShouldNot(Equal(0))
-		resetEvent := events[len(events)-1]
+		resetEvent := FindEventByName(events, eventgen.ClusterInstallationResetEventName)
+		Expect(resetEvent).NotTo(BeNil())
 		Expect(*resetEvent.Severity).Should(Equal(models.EventSeverityInfo))
 		Expect(*resetEvent.Message).Should(Equal("Reset cluster installation"))
 
@@ -1640,7 +1644,8 @@ var _ = Describe("ResetCluster", func() {
 		events, err := eventsHandler.V2GetEvents(ctx, c.ID, nil, nil)
 		Expect(err).ShouldNot(HaveOccurred())
 		Expect(len(events)).ShouldNot(Equal(0))
-		resetEvent := events[len(events)-1]
+		resetEvent := FindEventByName(events, eventgen.ResetInstallationFailedEventName)
+		Expect(resetEvent).NotTo(BeNil())
 		Expect(*resetEvent.Severity).Should(Equal(models.EventSeverityError))
 	})
 })
@@ -2836,24 +2841,26 @@ var _ = Describe("Deregister inactive clusters", func() {
 	})
 
 	It("Deregister inactive cluster limited", func() {
-		inactiveCluster1 := registerCluster()
-		inactiveCluster2 := registerCluster()
-		inactiveCluster3 := registerCluster()
-		inactiveCluster4 := registerCluster()
-		inactiveCluster5 := registerCluster()
-		inactiveCluster6 := registerCluster()
+		var (
+			clustersBeforeDeregisterInactive []*common.Cluster
+			clustersAfterDeregisterInactive  []*common.Cluster
+		)
+
+		for i := 0; i < 6; i++ {
+			registerCluster()
+		}
 
 		lastActive := strfmt.DateTime(time.Now())
 
-		Expect(state.DeregisterInactiveCluster(ctx, 3, lastActive)).ShouldNot(HaveOccurred())
+		err := db.Find(&clustersBeforeDeregisterInactive).Error
+		Expect(err).To(BeNil())
+		Expect(clustersBeforeDeregisterInactive).To(HaveLen(7))
 
-		Expect(wasDeregisterd(db, *inactiveCluster1.ID)).To(BeTrue())
-		Expect(wasDeregisterd(db, *inactiveCluster2.ID)).To(BeTrue())
+		Expect(state.DeregisterInactiveCluster(ctx, 3, lastActive)).ShouldNot(HaveOccurred())
 
-		Expect(wasDeregisterd(db, *inactiveCluster3.ID)).To(BeFalse())
-		Expect(wasDeregisterd(db, *inactiveCluster4.ID)).To(BeFalse())
-		Expect(wasDeregisterd(db, *inactiveCluster5.ID)).To(BeFalse())
-		Expect(wasDeregisterd(db, *inactiveCluster6.ID)).To(BeFalse())
+		err = db.Find(&clustersAfterDeregisterInactive).Error
+		Expect(err).To(BeNil())
+		Expect(clustersAfterDeregisterInactive).To(HaveLen(4))
 	})
 })
 
@@ -3663,3 +3670,12 @@ var _ = Describe("detectAndStoreCollidingIPsForCluster(cluster *common.Cluster,
 		Expect(expectedCollision[1]).To(BeEquivalentTo("de:ad:be:ef:00:00"))
 	})
 })
+
+func FindEventByName(events []*common.Event, eventName string) *common.Event {
+	for _, event := range events {
+		if event.Name == eventName {
+			return event
+		}
+	}
+	return nil
+}

internal/cluster/transition_test.go

@@ -284,11 +284,21 @@ var _ = Describe("Transition tests", func() {
 			},
 		}
 
+		findEventByMessage := func(events []*common.Event, eventMessage string) *common.Event {
+			for _, event := range events {
+				if swag.StringValue(event.Message) == eventMessage {
+					return event
+				}
+			}
+			return nil
+		}
+
 		checkCompleteInstallationUpdate := func(eventSeverity string, eventMessage string) {
 			events, err := eventsHandler.V2GetEvents(ctx, &clusterId, nil, nil)
 			Expect(err).ShouldNot(HaveOccurred())
 			Expect(len(events)).ShouldNot(Equal(0))
-			resetEvent := events[len(events)-1]
+			resetEvent := findEventByMessage(events, eventMessage)
+			Expect(resetEvent).ToNot(BeNil())
 			Expect(*resetEvent.Severity).Should(Equal(eventSeverity))
 
 			if eventMessage != "" {

internal/host/host_test.go

@@ -757,7 +757,8 @@ var _ = Describe("cancel installation", func() {
 			events, err := eventsHandler.V2GetEvents(ctx, h.ClusterID, h.ID, nil)
 			Expect(err).ShouldNot(HaveOccurred())
 			Expect(len(events)).ShouldNot(Equal(0))
-			cancelEvent := events[len(events)-1]
+			cancelEvent := FindEventByName(events, eventgen.HostInstallationCancelledEventName)
+			Expect(cancelEvent).NotTo(BeNil())
 			Expect(*cancelEvent.Severity).Should(Equal(models.EventSeverityInfo))
 			eventMessage := fmt.Sprintf("Installation cancelled for host %s", hostutil.GetHostnameForMsg(&h))
 			Expect(*cancelEvent.Message).Should(Equal(eventMessage))
@@ -776,7 +777,8 @@ var _ = Describe("cancel installation", func() {
 			events, err := eventsHandler.V2GetEvents(ctx, h.ClusterID, h.ID, nil)
 			Expect(err).ShouldNot(HaveOccurred())
 			Expect(len(events)).ShouldNot(Equal(0))
-			cancelEvent := events[len(events)-1]
+			cancelEvent := FindEventByName(events, eventgen.HostInstallationCancelledEventName)
+			Expect(cancelEvent).NotTo(BeNil())
 			Expect(*cancelEvent.Severity).Should(Equal(models.EventSeverityInfo))
 			eventMessage := fmt.Sprintf("Installation cancelled for host %s", hostutil.GetHostnameForMsg(&h))
 			Expect(*cancelEvent.Message).Should(Equal(eventMessage))
@@ -849,7 +851,8 @@ var _ = Describe("reset host", func() {
 			events, err := eventsHandler.V2GetEvents(ctx, h.ClusterID, h.ID, nil)
 			Expect(err).ShouldNot(HaveOccurred())
 			Expect(len(events)).ShouldNot(Equal(0))
-			resetEvent := events[len(events)-1]
+			resetEvent := FindEventByName(events, eventgen.HostInstallationResetEventName)
+			Expect(resetEvent).NotTo(BeNil())
 			Expect(*resetEvent.Severity).Should(Equal(models.EventSeverityInfo))
 			eventMessage := fmt.Sprintf("Installation reset for host %s", hostutil.GetHostnameForMsg(&h))
 			Expect(*resetEvent.Message).Should(Equal(eventMessage))
@@ -887,7 +890,8 @@ var _ = Describe("reset host", func() {
 			events, err := eventsHandler.V2GetEvents(ctx, &clusterId, h.ID, nil)
 			Expect(err).ShouldNot(HaveOccurred())
 			Expect(len(events)).ShouldNot(Equal(0))
-			resetEvent := events[len(events)-1]
+			resetEvent := FindEventByName(events, eventgen.UserRequiredCompleteInstallationResetEventName)
+			Expect(resetEvent).NotTo(BeNil())
 			Expect(*resetEvent.Severity).Should(Equal(models.EventSeverityInfo))
 			eventMessage := fmt.Sprintf("User action is required in order to complete installation reset for host %s", hostutil.GetHostnameForMsg(&h))
 			Expect(*resetEvent.Message).Should(Equal(eventMessage))
@@ -913,7 +917,8 @@ var _ = Describe("reset host", func() {
 			events, err := eventsHandler.V2GetEvents(ctx, h.ClusterID, h.ID, nil)
 			Expect(err).ShouldNot(HaveOccurred())
 			Expect(len(events)).ShouldNot(Equal(0))
-			resetEvent := events[len(events)-1]
+			resetEvent := FindEventByName(events, eventgen.UserRequiredCompleteInstallationResetEventName)
+			Expect(resetEvent).NotTo(BeNil())
 			Expect(*resetEvent.Severity).Should(Equal(models.EventSeverityInfo))
 			eventMessage := fmt.Sprintf("User action is required in order to complete installation reset for host %s", hostutil.GetHostnameForMsg(&h))
 			Expect(*resetEvent.Message).Should(Equal(eventMessage))
@@ -4723,3 +4728,12 @@ var _ = Describe("Rebooting day2", func() {
 	})
 
 })
+
+func FindEventByName(events []*common.Event, eventName string) *common.Event {
+	for _, event := range events {
+		if event.Name == eventName {
+			return event
+		}
+	}
+	return nil
+}

internal/migrations/20211115162400_modify_events_id_test.go

@@ -24,15 +24,24 @@ var _ = Describe("ModifyEventsID", func() {
 		events      []*common.Event
 	)
 
+	eventIsInList := func(eventId uint, events []*common.Event) bool {
+		for _, event := range events {
+			if event.ID == eventId {
+				return true
+			}
+		}
+		return false
+	}
+
 	validateEvents := func() {
 		validationDB, err := common.OpenTestDBConn(dbName)
 		Expect(err).ShouldNot(HaveOccurred())
 		defer common.CloseDB(validationDB)
 		var eventsAfterMigrate []*common.Event
 		Expect(validationDB.Find(&eventsAfterMigrate).Error).ToNot(HaveOccurred())
 		Expect(eventsAfterMigrate).To(HaveLen(2))
-		Expect(events[0].ID).To(Equal(eventsAfterMigrate[0].ID))
-		Expect(events[1].ID).To(Equal(eventsAfterMigrate[1].ID))
+		Expect(eventIsInList(events[0].ID, eventsAfterMigrate)).To(BeTrue())
+		Expect(eventIsInList(events[1].ID, eventsAfterMigrate)).To(BeTrue())
 	}
 
 	BeforeEach(func() {

internal/migrations/20211125182200_change_hosts_pkey_test.go

@@ -19,15 +19,24 @@ var _ = Describe("Migrate hosts pkey", func() {
 		hosts  []*common.Host
 	)
 
+	hostIsInList := func(hostId *strfmt.UUID, hosts []*common.Host) bool {
+		for _, host := range hosts {
+			if host.ID.String() == hostId.String() {
+				return true
+			}
+		}
+		return false
+	}
+
 	validateHosts := func() {
 		validationDB, err := common.OpenTestDBConn(dbName)
 		Expect(err).ShouldNot(HaveOccurred())
 		defer common.CloseDB(validationDB)
 		var hostsAfterMigrate []*common.Host
 		Expect(validationDB.Find(&hostsAfterMigrate).Error).ToNot(HaveOccurred())
 		Expect(hostsAfterMigrate).To(HaveLen(2))
-		Expect(hosts[0].ID).To(Equal(hostsAfterMigrate[0].ID))
-		Expect(hosts[1].ID).To(Equal(hostsAfterMigrate[1].ID))
+		Expect(hostIsInList(hosts[0].ID, hostsAfterMigrate)).To(BeTrue())
+		Expect(hostIsInList(hosts[1].ID, hostsAfterMigrate)).To(BeTrue())
 	}
 
 	BeforeEach(func() {