diff --git a/contributes.json b/contributes.json new file mode 100644 index 000000000..67c09256a --- /dev/null +++ b/contributes.json @@ -0,0 +1,10 @@ +{ + "contributes": { + "jsonValidation": [ + { + "fileMatch": "migrations/*.yaml", + "url": "migrations/.schema.json" + } + ] + } +} diff --git a/docs/_sidebar.md b/docs/_sidebar.md index 61133af91..b59a77d7e 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -7,6 +7,7 @@ - [Debug Console](flagr_debugging.md) - Server Configuration - [Env](flagr_env.md) + - [Migrations](flagr_migrations.md) - Client SDKs - [Ruby SDK 🔗](https://github.com/openflagr/rbflagr) - [Go SDK 🔗](https://github.com/openflagr/goflagr) diff --git a/docs/flagr_migrations.md b/docs/flagr_migrations.md new file mode 100644 index 000000000..08fc64592 --- /dev/null +++ b/docs/flagr_migrations.md @@ -0,0 +1,53 @@ +# Migrations + +Users can create flag collections as yaml files within a migration directory and run flagr to insert/modify flags. +This allows for developers to create flags as deployment assets and allows for migrating flags through various environments whilst keeping keys stable. + +Each found flag is upserted into the database. Then the flag has all segments, variants and tags removed then replaced with the new flag properties. + +As an example, create a file named `migrations/202403030000.yaml` with the following content: +```yaml +--- +# this is a basic flag +- key: SIMPLE-FLAG-1 + description: a toggle for just one user + enabled: true + segments: + - description: flag for just for one email test@test.com + rank: 0 + rolloutPercent: 100 + constraints: + - property: email + operator: EQ + value: '"test@test.com"' + distributions: + - variantKey: "on" + percent: 100 + - rank: 1 + rolloutPercent: 100 + constraints: [] + distributions: + - variantKey: "off" + percent: 100 + variants: + - key: "off" + attachment: {} + - key: "on" + attachment: {} + entityType: User + dataRecordsEnabled: true + +``` + +```shell +$ flagr -m +INFO[0146] 1 new migrations completed (1 total) +``` +Once the application is ran, flagr will scan the migration files, insert them into the db and shut down. + +### Config +Location of yaml configs can be set with either argument or env var. +``` +FLAGR_MIGRATION_PATH=./migrations/ ./flagr -m +./flagr -m --migrationPath=`pwd`/migrations +``` diff --git a/go.mod b/go.mod index 9f5fc4bbc..650e4c31c 100644 --- a/go.mod +++ b/go.mod @@ -147,4 +147,5 @@ require ( modernc.org/mathutil v1.5.0 // indirect modernc.org/memory v1.5.0 // indirect modernc.org/sqlite v1.23.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 0fe82157d..3717e177a 100644 --- a/go.sum +++ b/go.sum @@ -714,3 +714,5 @@ modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw modernc.org/tcl v1.15.0/go.mod h1:xRoGotBZ6dU+Zo2tca+2EqVEeMmOUBzHnhIwq4YrVnE= modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/z v1.7.0/go.mod h1:hVdgNMh8ggTuRG1rGU8x+xGRFfiQUIAw0ZqlPy8+HyQ= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/migrations/.schema.json b/migrations/.schema.json new file mode 100644 index 000000000..e4c5ffa1f --- /dev/null +++ b/migrations/.schema.json @@ -0,0 +1,125 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "description": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "segments": { + "type": "array", + "items": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "rank": { + "type": "integer" + }, + "rolloutPercent": { + "type": "integer" + }, + "constraints": { + "type": "array", + "items": { + "type": "object", + "properties": { + "property": { + "type": "string" + }, + "operator": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "property", + "operator", + "value" + ] + } + }, + "distributions": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "variantKey": { + "type": "string" + }, + "percent": { + "type": "integer" + } + }, + "required": [ + "variantKey", + "percent" + ] + } + ] + } + }, + "required": [ + "rank", + "rolloutPercent", + "distributions" + ] + } + }, + "variants": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "attachment": { + "type": "object" + } + }, + "required": [ + "key" + ] + } + }, + "entityType": { + "type": "string" + }, + "dataRecordsEnabled": { + "type": "boolean" + }, + "tags": { + "type": "array", + "items": + { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ] + } + } + }, + "required": [ + "key", + "description", + "enabled" + ] + } +} diff --git a/pkg/entity/db.go b/pkg/entity/db.go index a29150b96..2b48ca246 100644 --- a/pkg/entity/db.go +++ b/pkg/entity/db.go @@ -32,6 +32,7 @@ var AutoMigrateTables = []interface{}{ Variant{}, Tag{}, FlagEntityType{}, + FlagMigration{}, } func connectDB() (db *gorm.DB, err error) { diff --git a/pkg/entity/flag_migration.go b/pkg/entity/flag_migration.go new file mode 100644 index 000000000..22c287100 --- /dev/null +++ b/pkg/entity/flag_migration.go @@ -0,0 +1,9 @@ +package entity + +import "gorm.io/gorm" + +type FlagMigration struct { + gorm.Model + + Name string `gorm:"type:varchar(64);uniqueIndex:idx_flag_migration_name"` +} diff --git a/pkg/handler/flag_migrations.go b/pkg/handler/flag_migrations.go new file mode 100644 index 000000000..9bb97b763 --- /dev/null +++ b/pkg/handler/flag_migrations.go @@ -0,0 +1,194 @@ + +package handler + +import ( + "github.com/openflagr/flagr/pkg/entity" + "github.com/openflagr/flagr/pkg/mapper/entity_restapi/r2e" + "github.com/openflagr/flagr/pkg/util" + "github.com/openflagr/flagr/swagger_gen/models" + "github.com/sirupsen/logrus" + "github.com/spf13/cast" + "gorm.io/gorm/clause" + "os" + "sigs.k8s.io/yaml" + "strings" +) + +type FlagMigrationOptions struct { + Run bool `long:"migrations" short:"m" description:"Run Flag Migrations and Exit"` + Path string `long:"migrationPath" description:"Migration files path" env:"FLAGR_MIGRATION_PATH"` +} + +func FlagMigrations(path string) error { + return migrateFromDir(path) +} + +func migrateFromDir(dir string) error { + tx := getDB() + fms := []entity.FlagMigration{} + + files, err := os.ReadDir(dir) + if err != nil { + logrus.WithField("err", err).Errorf("cannot read directory for migrations: %v", err) + return err + } + + completedFiles := make(map[string]bool) + + tx.Find(&fms) + + for _, fm := range fms { + completedFiles[fm.Name] = true + } + + completed := 0 + for _, file := range files { + filename := file.Name() + if strings.HasSuffix(filename, ".yaml") && !completedFiles[filename] { + flags, err := readMigrationFile(dir, filename) + if err != nil { + continue + } + + completed = processYamlFile(flags, filename) + completed + } + + } + + logrus.Infof("%d new migrations completed", completed) + return nil +} + +func processYamlFile(flags []models.Flag, filename string) int { + for _, flagModel := range flags { + if f, err := migrateFlag(flagModel); err == nil { + entity.SaveFlagSnapshot(getDB(), util.SafeUint(f.ID), "migration "+filename) + } + } + + fm := entity.FlagMigration{Name: filename} + getDB().Create(&fm) + return 1 +} + +func migrateFlag(flagModel models.Flag) (*entity.Flag, error) { + flag := &entity.Flag{ + EntityType: flagModel.EntityType, + Key: flagModel.Key, + Description: util.SafeString(flagModel.Description), + DataRecordsEnabled: cast.ToBool(flagModel.DataRecordsEnabled), + Enabled: cast.ToBool(flagModel.Enabled), + Notes: util.SafeStringWithDefault(flagModel.Notes, ""), + } + + tx := getDB() + + //upsert + tx.Where(entity.Flag{Key: flagModel.Key}).Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "key"}}, + UpdateAll: true, + }).Create(&flag) + + tx = entity.PreloadSegmentsVariantsTags(getDB()) + + tx.Where(entity.Flag{Key: flagModel.Key}).First(&flag) + + // delete the association + deleteTags(flag) + + // delete the entity (keeps associations + deleteVariants(flag) + deleteSegments(flag) + + addTags(flagModel, flag) + + // used to map variant to distribution + variantMap := saveVariants(flagModel, flag) + + saveSegments(flagModel, flag, variantMap) + + return flag, nil +} + +func saveSegments(flagModel models.Flag, flag *entity.Flag, variantMap map[string]int64) { + for _, segmentModel := range flagModel.Segments { + segment := entity.Segment{ + Description: util.SafeString(segmentModel.Description), + RolloutPercent: uint(*segmentModel.RolloutPercent), + Rank: uint(*segmentModel.Rank), + } + // save segment to flag + getDB().Model(flag).Association("Segments").Append(&segment) + // map distribution to variant + for _, distribution := range segmentModel.Distributions { + variantId := variantMap[util.SafeString(distribution.VariantKey)] + distribution.VariantID = &variantId + } + segment.Distributions = r2e.MapDistributions(segmentModel.Distributions, segment.ID) + segment.Constraints = r2e.MapConstraints(segmentModel.Constraints, segment.ID) + getDB().Save(&segment) + } +} + +func saveVariants(flagModel models.Flag, flag *entity.Flag) map[string]int64 { + variantMap := make(map[string]int64) + + // add variants + for _, variantModel := range flagModel.Variants { + a, _ := r2e.MapAttachment(variantModel.Attachment) + variant := entity.Variant{ + Key: util.SafeString(variantModel.Key), + Attachment: a, + } + getDB().Model(flag).Association("Variants").Append(&variant) + variantMap[util.SafeString(variant.Key)] = int64(variant.ID) + } + return variantMap +} + +func addTags(flagModel models.Flag, flag *entity.Flag) { + // add tags + for _, tagModel := range flagModel.Tags { + t := &entity.Tag{} + t.Value = util.SafeString(tagModel.Value) + getDB().Where("value = ?", util.SafeString(tagModel.Value)).Find(t) + getDB().Model(flag).Association("Tags").Append(t) + } +} + +func deleteSegments(flag *entity.Flag) { + for _, segmentsModel := range flag.Segments { + getDB().Select("Constraints", "Distributions").Delete(&entity.Segment{}, segmentsModel.ID) + } +} + +func deleteVariants(flag *entity.Flag) { + for _, variantsModel := range flag.Variants { + v:= &entity.Variant{} + v.ID = variantsModel.ID + getDB().Delete(&entity.Variant{}, variantsModel.ID) + } +} + +func deleteTags(flag *entity.Flag) { + for _, tagsModel := range flag.Tags { + t := &entity.Tag{} + t.ID = uint(tagsModel.ID) + getDB().Model(flag).Association("Tags").Delete(t) + } +} + +func readMigrationFile(dir string, fileName string) ([]models.Flag, error) { + f := dir + string(os.PathSeparator) + fileName + contents, err := os.ReadFile(f) + if err != nil { + return nil, err + } + + var flags []models.Flag + err = yaml.Unmarshal([]byte(contents), &flags) + if err != nil { + return nil, err + } + return flags, nil +} diff --git a/pkg/handler/flag_migrations_test.go b/pkg/handler/flag_migrations_test.go new file mode 100644 index 000000000..331a3a03b --- /dev/null +++ b/pkg/handler/flag_migrations_test.go @@ -0,0 +1,31 @@ +package handler + +import ( + "github.com/openflagr/flagr/pkg/entity" + "github.com/prashantv/gostub" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestFlagMigrations(t *testing.T) { + db := entity.NewTestDB() + + tmpDB, dbErr := db.DB() + if dbErr != nil { + t.Errorf("Failed to get database") + } + + defer tmpDB.Close() + + defer gostub.StubFunc(&getDB, db).Reset() + + t.Run("FlagMigrationWithValidPath", func(t *testing.T) { + err := FlagMigrations("./testdata/migrations") + assert.Nil(t, err) + }) + + t.Run("FlagMigrationWithInvalidPath", func(t *testing.T) { + err := FlagMigrations("./testdata/migration") + assert.NotNil(t, err) + }) +} diff --git a/pkg/handler/testdata/migrations/test-migration-broken.yaml b/pkg/handler/testdata/migrations/test-migration-broken.yaml new file mode 100644 index 000000000..4bfac2958 --- /dev/null +++ b/pkg/handler/testdata/migrations/test-migration-broken.yaml @@ -0,0 +1,7 @@ +--- +# this is a broken yaml since enabled should be a boolean +- key: FLAG-111 + description: basic flag + entityType: User + dataRecordsEnabled: false + enabled: 9 diff --git a/pkg/handler/testdata/migrations/test-migration.yaml b/pkg/handler/testdata/migrations/test-migration.yaml new file mode 100644 index 000000000..cbc271e4a --- /dev/null +++ b/pkg/handler/testdata/migrations/test-migration.yaml @@ -0,0 +1,43 @@ +--- +# this is a basic +- key: FLAGS-123 + description: a flag added by migration + segments: + - description: just email + rank: 0 + rolloutPercent: 100 + constraints: + - property: email + operator: EQ + value: '"me@me.com"' + distributions: + - variantKey: "on" + percent: 100 + - rank: 1 + rolloutPercent: 100 + distributions: + - variantKey: "off" + percent: 100 + variants: + - key: "off" + - key: "on" + entityType: User + dataRecordsEnabled: true + enabled: false + tags: + - value: "better tags" +- key: FLAGS-123 + description: updated second time + segments: + - rank: 1 + rolloutPercent: 100 + distributions: + - variantKey: "off" + percent: 100 + variants: + - key: "off" + entityType: User + dataRecordsEnabled: false + enabled: true + tags: + - value: "best tag" diff --git a/pkg/mapper/entity_restapi/r2e/r2e.go b/pkg/mapper/entity_restapi/r2e/r2e.go index a4e169e8f..577b692e5 100644 --- a/pkg/mapper/entity_restapi/r2e/r2e.go +++ b/pkg/mapper/entity_restapi/r2e/r2e.go @@ -29,6 +29,25 @@ func MapDistribution(r *models.Distribution, segmentID uint) entity.Distribution return e } +func MapConstraints(r []*models.Constraint, segmentID uint) []entity.Constraint { + e := make([]entity.Constraint, len(r)) + for i, d := range r { + e[i] = MapConstraint(d, segmentID) + } + return e +} + +// MapDistribution maps distribution +func MapConstraint(r *models.Constraint, segmentID uint) entity.Constraint { + e := entity.Constraint{ + SegmentID: segmentID, + Property: util.SafeString(r.Property), + Operator: util.SafeString(r.Operator), + Value: util.SafeString(r.Value), + } + return e +} + // MapAttachment maps attachment func MapAttachment(a interface{}) (entity.Attachment, error) { e := entity.Attachment{} diff --git a/swagger_gen/restapi/configure_flagr.go b/swagger_gen/restapi/configure_flagr.go index f29802cd7..2288e6d25 100644 --- a/swagger_gen/restapi/configure_flagr.go +++ b/swagger_gen/restapi/configure_flagr.go @@ -4,9 +4,11 @@ package restapi import ( "crypto/tls" + "github.com/go-openapi/swag" jsoniter "github.com/json-iterator/go" - "net/http" "io" + "net/http" + "os" "github.com/openflagr/flagr/pkg/config" "github.com/openflagr/flagr/pkg/handler" @@ -21,8 +23,19 @@ import ( //go:generate swagger generate server --target ../../swagger_gen --name Flagr --spec ../../docs/api_docs/bundle.yaml +var flagMigrationOptions = handler.FlagMigrationOptions{ + Run: false, + Path: "./migrations", +} + func configureFlags(api *operations.FlagrAPI) { - // api.CommandLineOptionsGroups = []swag.CommandLineOptionsGroup{ ... } + api.CommandLineOptionsGroups = []swag.CommandLineOptionsGroup{ + swag.CommandLineOptionsGroup{ + ShortDescription: "Startup", + Options: &flagMigrationOptions, + }, + } + } func configureAPI(api *operations.FlagrAPI) http.Handler { @@ -60,6 +73,13 @@ func configureTLS(tlsConfig *tls.Config) { // This function can be called multiple times, depending on the number of serving schemes. // scheme value will be set accordingly: "http", "https" or "unix" func configureServer(s *http.Server, scheme, addr string) { + if flagMigrationOptions.Run { + if err:= handler.FlagMigrations(flagMigrationOptions.Path); err != nil { + os.Exit(1) + } else { + os.Exit(0) + } + } } // The middleware configuration is for the handler executors. These do not apply to the swagger.json document.