feat(shipyard-controller): Store sequence executions in new format wi…

…thout potential dots (.) in property names (#7605)

* introduce db model for sequence executions

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

* transformation functions for sequence execution state

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

* added unit test

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

* also store task properties as json string

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

* conversion between sequence execution schemas

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

* added methods to decode back from internal representation of sequence executions

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

* added unit tests

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

* added unit tests

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

* unit tests

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

* added model transformer option for sequence execution repo

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

* added tests for sequence execution migrator

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

* rename properties that are encoded differently

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

* run sequence execution migrator

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

* replace items during migration to prevent old properties from being kept

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

* trigger build

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

* additional unit tests

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

* fixed unit tests

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

shipyard-controller/db/migration/project_mv_migration_test.go

@@ -22,11 +22,13 @@ func setupLocalMongoDB() func() {
 	mongoServer, err := memongo.Start(mongoDbVersion)
 	randomDbName := memongo.RandomDatabase()
 
+	mongoURL := fmt.Sprintf("%s/%s", mongoServer.URI(), randomDbName)
 	os.Setenv("MONGODB_DATABASE", randomDbName)
-	os.Setenv("MONGODB_EXTERNAL_CONNECTION_STRING", fmt.Sprintf("%s/%s", mongoServer.URI(), randomDbName))
+	os.Setenv("MONGODB_EXTERNAL_CONNECTION_STRING", mongoURL)
 
 	var mongoDBClient *mongo.Client
 	mongoDBClient, err = mongo.NewClient(options.Client().ApplyURI(mongoServer.URI()))
+	logger.Infof("MongoDB Server runnning at: %s", mongoURL)
 	if err != nil {
 		logger.Fatalf("Mongo Client setup failed: %s", err)
 	}

shipyard-controller/db/migration/sequence_execution_migration.go

@@ -0,0 +1,67 @@
+package migration
+
+import (
+	"fmt"
+	apimodels "github.com/keptn/go-utils/pkg/api/models"
+	keptnv2 "github.com/keptn/go-utils/pkg/lib/v0_2_0"
+	"github.com/keptn/keptn/shipyard-controller/db"
+	v1 "github.com/keptn/keptn/shipyard-controller/db/models/sequence_execution/v1"
+	"github.com/keptn/keptn/shipyard-controller/models"
+	logger "github.com/sirupsen/logrus"
+)
+
+// NewSequenceExecutionMigrator creates a new SequenceExecutionMigrator
+// Internally it is using the SequenceExecutionJsonStringRepo decorator
+// which stores the arbitrary event payload sent by keptn integrations as Json strings to avoid having property names with dots (.) in them
+func NewSequenceExecutionMigrator(dbConnection *db.MongoDBConnection) *SequenceExecutionMigrator {
+	return &SequenceExecutionMigrator{
+		projectRepo:           db.NewMongoDBKeyEncodingProjectsRepo(dbConnection),
+		sequenceExecutionRepo: db.NewMongoDBSequenceExecutionRepo(dbConnection, db.WithSequenceExecutionModelTransformer(&v1.ModelTransformer{})),
+	}
+}
+
+type SequenceExecutionMigrator struct {
+	sequenceExecutionRepo db.SequenceExecutionRepo
+	projectRepo           db.ProjectRepo
+}
+
+// Run retrieves all existing sequence executions from the repository
+// and performs an update operation on each of them using the SequenceExecutionJsonStringRepo.
+// This way, sequence executions containing stored with the previous format are migrated to the new one
+func (s *SequenceExecutionMigrator) Run() error {
+	projects, err := s.projectRepo.GetProjects()
+	if err != nil {
+		return fmt.Errorf("could not migrate sequence executions to new format: %w", err)
+	}
+	s.updateSequenceExecutionsOfProject(projects)
+	return nil
+}
+
+func (s *SequenceExecutionMigrator) updateSequenceExecutionsOfProject(projects []*apimodels.ExpandedProject) {
+	if projects == nil {
+		return
+	}
+	for _, project := range projects {
+		sequenceExecutions, err := s.sequenceExecutionRepo.Get(models.SequenceExecutionFilter{
+			Scope: models.EventScope{
+				EventData: keptnv2.EventData{
+					Project: project.ProjectName,
+				},
+			}})
+		if err != nil {
+			logger.Errorf("Could not retrieve sequence executions for project %s: %v", project.ProjectName, err)
+			continue
+		}
+		for _, sequenceExecution := range sequenceExecutions {
+			if sequenceExecution.SchemaVersion == v1.SchemaVersionV1 {
+				continue
+			}
+			if err := s.sequenceExecutionRepo.Upsert(sequenceExecution, &models.SequenceExecutionUpsertOptions{
+				Replace: true,
+			}); err != nil {
+				logger.Errorf("Could not update sequence execution with ID %s for project %s: %v", sequenceExecution.ID, project.ProjectName, err)
+				continue
+			}
+		}
+	}
+}