Permalink
Browse files

init, pull: Always store default-character-set and default-collation

Previously, in schema-level .skeema files, init and pull only populated the
default-character-set and default-collation options if the schema-level values
differed from the server's global defaults. This is brittle, as the server
global default is not tracked by Skeema, and the default server global values
have changed in MySQL 8.0. As of this commit, the init command will always
populate the values for each schema; the pull command will now fill in missing
values for existing schemas.
  • Loading branch information...
evanelias committed Aug 24, 2018
1 parent 796de4c commit fe2caef5e1c3d4c5a2f61e4cbd272344868a9554
Showing with 89 additions and 54 deletions.
  1. +6 −20 cmd_init.go
  2. +4 −24 cmd_pull.go
  3. +2 −6 doc/options.md
  4. +1 −1 skeema.go
  5. +1 −1 skeema_cmd_test.go
  6. +2 −2 skeema_test.go
  7. +2 −0 testdata/golden-mariadb102/autoinc/mydb/analytics/.skeema
  8. +2 −0 testdata/golden-mariadb102/autoinc/mydb/product/.skeema
  9. +2 −0 testdata/golden-mariadb102/ignore/mydb/analytics/.skeema
  10. +2 −0 testdata/golden-mariadb102/ignore/mydb/product/.skeema
  11. +2 −0 testdata/golden-mariadb102/init/mydb/analytics/.skeema
  12. +2 −0 testdata/golden-mariadb102/init/mydb/product/.skeema
  13. +2 −0 testdata/golden-mariadb102/pull1/mydb/product/.skeema
  14. +2 −0 testdata/golden-mariadb102/unsupported/mydb/analytics/.skeema
  15. +2 −0 testdata/golden-mariadb102/unsupported/mydb/product/.skeema
  16. +2 −0 testdata/golden-mysql55/autoinc/mydb/analytics/.skeema
  17. +2 −0 testdata/golden-mysql55/autoinc/mydb/product/.skeema
  18. +2 −0 testdata/golden-mysql55/ignore/mydb/analytics/.skeema
  19. +2 −0 testdata/golden-mysql55/ignore/mydb/product/.skeema
  20. +2 −0 testdata/golden-mysql55/init/mydb/analytics/.skeema
  21. +2 −0 testdata/golden-mysql55/init/mydb/product/.skeema
  22. +2 −0 testdata/golden-mysql55/pull1/mydb/product/.skeema
  23. +2 −0 testdata/golden-mysql55/unsupported/mydb/analytics/.skeema
  24. +2 −0 testdata/golden-mysql55/unsupported/mydb/product/.skeema
  25. +2 −0 testdata/golden-mysql80/autoinc/mydb/analytics/.skeema
  26. +2 −0 testdata/golden-mysql80/autoinc/mydb/product/.skeema
  27. +2 −0 testdata/golden-mysql80/ignore/mydb/analytics/.skeema
  28. +2 −0 testdata/golden-mysql80/ignore/mydb/product/.skeema
  29. +2 −0 testdata/golden-mysql80/init/mydb/analytics/.skeema
  30. +2 −0 testdata/golden-mysql80/init/mydb/product/.skeema
  31. +1 −0 testdata/golden-mysql80/pull1/mydb/archives/.skeema
  32. +2 −0 testdata/golden-mysql80/pull1/mydb/product/.skeema
  33. +2 −0 testdata/golden-mysql80/unsupported/mydb/analytics/.skeema
  34. +2 −0 testdata/golden-mysql80/unsupported/mydb/product/.skeema
  35. +2 −0 testdata/golden/autoinc/mydb/analytics/.skeema
  36. +2 −0 testdata/golden/autoinc/mydb/product/.skeema
  37. +2 −0 testdata/golden/ignore/mydb/analytics/.skeema
  38. +2 −0 testdata/golden/ignore/mydb/product/.skeema
  39. +2 −0 testdata/golden/init/mydb/analytics/.skeema
  40. +2 −0 testdata/golden/init/mydb/product/.skeema
  41. +2 −0 testdata/golden/pull1/mydb/product/.skeema
  42. +2 −0 testdata/golden/unsupported/mydb/analytics/.skeema
  43. +2 −0 testdata/golden/unsupported/mydb/product/.skeema
View
@@ -106,12 +106,6 @@ func InitHandler(cfg *mybase.Config) error {
return NewExitValue(CodeBadConfig, "Schema %s does not exist on instance %s", onlySchema, inst)
}
// Look up server charset and collation, so that we know which schemas override
instCharSet, instCollation, err := inst.DefaultCharSetAndCollation()
if err != nil {
return err
}
// Figure out what needs to go in the hostDir's .skeema file.
hostOptionFile := mybase.NewFile(hostDir.Path, ".skeema")
hostOptionFile.SetOptionValue(environment, "host", inst.Host)
@@ -136,12 +130,8 @@ func InitHandler(cfg *mybase.Config) error {
// schema name is placed outside of any named section/environment since the
// default assumption is that schema names match between environments
hostOptionFile.SetOptionValue("", "schema", onlySchema)
if instCharSet != schemas[0].CharSet {
hostOptionFile.SetOptionValue("", "default-character-set", schemas[0].CharSet)
}
if instCollation != schemas[0].Collation {
hostOptionFile.SetOptionValue("", "default-collation", schemas[0].Collation)
}
hostOptionFile.SetOptionValue("", "default-character-set", schemas[0].CharSet)
hostOptionFile.SetOptionValue("", "default-collation", schemas[0].Collation)
}
// Write the option file
@@ -161,7 +151,7 @@ func InitHandler(cfg *mybase.Config) error {
// Iterate over the schemas. For each one, create a dir with .skeema and *.sql files
for _, s := range schemas {
if err := PopulateSchemaDir(s, hostDir, separateSchemaSubdir, instCharSet != s.CharSet, instCollation != s.Collation); err != nil {
if err := PopulateSchemaDir(s, hostDir, separateSchemaSubdir); err != nil {
return err
}
}
@@ -175,7 +165,7 @@ func InitHandler(cfg *mybase.Config) error {
// *.sql files will be put in parentDir, and it will be the caller's
// responsibility to ensure its .skeema option file exists and maps to the
// correct schema name.
func PopulateSchemaDir(s *tengo.Schema, parentDir *Dir, makeSubdir, includeCharSet, includeCollation bool) error {
func PopulateSchemaDir(s *tengo.Schema, parentDir *Dir, makeSubdir bool) error {
// Ignore any attempt to populate a dir for the temp schema
if s.Name == parentDir.Config.Get("temp-schema") {
return nil
@@ -196,12 +186,8 @@ func PopulateSchemaDir(s *tengo.Schema, parentDir *Dir, makeSubdir, includeCharS
// names match between environments.
optionFile := mybase.NewFile(".skeema")
optionFile.SetOptionValue("", "schema", s.Name)
if includeCharSet {
optionFile.SetOptionValue("", "default-character-set", s.CharSet)
}
if includeCollation {
optionFile.SetOptionValue("", "default-collation", s.Collation)
}
optionFile.SetOptionValue("", "default-character-set", s.CharSet)
optionFile.SetOptionValue("", "default-collation", s.Collation)
if schemaDir, err = parentDir.CreateSubdir(s.Name, optionFile); err != nil {
return NewExitValue(CodeCantCreate, "Unable to use directory %s for schema %s: %s", path.Join(parentDir.Path, s.Name), s.Name, err)
}
View
@@ -64,24 +64,12 @@ func PullHandler(cfg *mybase.Config) error {
// Handle changes in schema's default character set and/or collation by
// persisting changes to the dir's option file. File operation errors here
// are just surfaced as warnings.
if diff.SchemaDDL != "" {
instCharSet, instCollation, err := t.Instance.DefaultCharSetAndCollation()
if err != nil {
return err
}
if diff.SchemaDDL != "" || t.Dir.Config.Get("default-character-set") == "" || t.Dir.Config.Get("default-collation") == "" {
if optionFile, err := t.Dir.OptionFile(); err != nil {
log.Warnf("Unable to update character set and/or collation for %s/.skeema: %s", t.Dir, err)
} else {
if instCharSet != t.SchemaFromInstance.CharSet {
optionFile.SetOptionValue("", "default-character-set", t.SchemaFromInstance.CharSet)
} else {
optionFile.UnsetOptionValue("", "default-character-set")
}
if instCollation != t.SchemaFromInstance.Collation {
optionFile.SetOptionValue("", "default-collation", t.SchemaFromInstance.Collation)
} else {
optionFile.UnsetOptionValue("", "default-collation")
}
optionFile.SetOptionValue("", "default-character-set", t.SchemaFromInstance.CharSet)
optionFile.SetOptionValue("", "default-collation", t.SchemaFromInstance.Collation)
if err = optionFile.Write(true); err != nil {
log.Warnf("Unable to update character set and/or collation for %s: %s", optionFile.Path(), err)
} else {
@@ -291,7 +279,6 @@ func findNewSchemas(dir *Dir) error {
// Compare dirs to schemas, UNLESS subdirs exist but don't actually map to schemas directly
if len(subdirHasSchema) > 0 || len(subdirs) == 0 {
var instCharSet, instCollation string
schemaNames, err := instance.SchemaNames()
if err != nil {
return err
@@ -303,15 +290,8 @@ func findNewSchemas(dir *Dir) error {
if err != nil {
return err
}
// Lazily fetch instance charset and collation upon first need
if instCharSet == "" {
instCharSet, instCollation, err = instance.DefaultCharSetAndCollation()
if err != nil {
return err
}
}
// use same logic from init command
if err := PopulateSchemaDir(s, dir, true, instCharSet != s.CharSet, instCollation != s.Collation); err != nil {
if err := PopulateSchemaDir(s, dir, true); err != nil {
return err
}
}
View
@@ -261,9 +261,7 @@ Commands | *all*
**Type** | string
**Restrictions** | Should only appear in a .skeema option file that also contains [schema](#schema)
This option specifies the default character set to use for a particular schema. Its name and purpose matches the corresponding option in MySQL `db.opt` files. In Skeema, this option is only needed in cases where some schemas have a different default character set than the server-level default.
When `skeema init` or `skeema pull` imports a schema subdirectory for the first time, or when `skeema pull` updates an existing schema subdirectory, the schema's default character set will be compared to the instance's server-level default character set. If they differ, [default-character-set](#default-character-set) will be populated in the subdir's .skeema file automatically. Otherwise, it will be omitted.
This option specifies the default character set to use for a particular schema. In .skeema files, it is populated automatically by `skeema init` and updated automatically by `skeema pull`.
If a new schema is being created for the first time via `skeema push`, and [default-character-set](#default-character-set) has been set, it will be included as part of the `CREATE DATABASE` statement. If it has not been set, the instance's default server-level character set is used instead.
@@ -277,9 +275,7 @@ Commands | *all*
**Type** | string
**Restrictions** | Should only appear in a .skeema option file that also contains [schema](#schema)
This option specifies the default collation to use for a particular schema. Its name and purpose matches the corresponding option in MySQL `db.opt` files. In Skeema, this option is only needed in cases where some schemas have a different default collation than the server-level default.
When `skeema init` or `skeema pull` imports a schema subdirectory for the first time, or when `skeema pull` updates an existing schema subdirectory, the schema's default collation will be compared to the instance's server-level default collation. If they differ, [default-collation](#default-collation) will be populated in the subdir's .skeema file automatically. Otherwise, it will be omitted.
This option specifies the default collation to use for a particular schema. In .skeema files, it is populated automatically by `skeema init` and updated automatically by `skeema pull`.
If a new schema is being created for the first time via `skeema push`, and [default-collation](#default-collation) has been set, it will be included as part of the `CREATE DATABASE` statement. If it has not been set, the instance's default server-level collation is used instead.
View
@@ -14,7 +14,7 @@ schema to the filesystem, and apply online schema changes by modifying files.`
// Globals overridden by GoReleaser's ldflags
var (
version = "1.0.3-dev"
version = "1.0.4-dev"
commit = "unknown"
date = "unknown"
)
View
@@ -164,7 +164,7 @@ func (s *SkeemaIntegrationSuite) TestPullHandler(t *testing.T) {
s.verifyFiles(t, cfg, "../golden/pull1")
// Revert db back to previous state, and pull again to test the opposite
// behaviors: delete dir for new schema, remove charset/collation from .skeema,
// behaviors: delete dir for new schema, restore charset/collation in .skeema,
// etc. Also edit the host .skeema file to remove flavor, to test logic that
// adds/updates flavor on pull.
s.cleanData(t, "setup.sql")
View
@@ -224,8 +224,8 @@ func (s *SkeemaIntegrationSuite) verifyFiles(t *testing.T, cfg *mybase.Config, d
if !aOptionFile.SameContents(bOptionFile) {
t.Errorf("File contents do not match between %s and %s", aOptionFile.Path(), bOptionFile.Path())
fmt.Println("Expected:\n", readFile(t, aOptionFile.Path()))
fmt.Println("Actual:\n", readFile(t, bOptionFile.Path()))
fmt.Printf("Expected:\n%s\n", readFile(t, aOptionFile.Path()))
fmt.Printf("Actual:\n%s\n", readFile(t, bOptionFile.Path()))
}
}
@@ -1 +1,3 @@
schema=analytics
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=product
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=analytics
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=product
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=analytics
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=product
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=product
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=analytics
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=product
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=analytics
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=product
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=analytics
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=product
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=analytics
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=product
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=product
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=analytics
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=product
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=analytics
default-character-set=utf8mb4
default-collation=utf8mb4_0900_ai_ci
@@ -1 +1,3 @@
schema=product
default-character-set=utf8mb4
default-collation=utf8mb4_0900_ai_ci
@@ -1 +1,3 @@
schema=analytics
default-character-set=utf8mb4
default-collation=utf8mb4_0900_ai_ci
@@ -1 +1,3 @@
schema=product
default-character-set=utf8mb4
default-collation=utf8mb4_0900_ai_ci
@@ -1 +1,3 @@
schema=analytics
default-character-set=utf8mb4
default-collation=utf8mb4_0900_ai_ci
@@ -1 +1,3 @@
schema=product
default-character-set=utf8mb4
default-collation=utf8mb4_0900_ai_ci
@@ -1,2 +1,3 @@
schema=archives
default-character-set=utf8mb4
default-collation=utf8mb4_general_ci
@@ -1 +1,3 @@
schema=product
default-character-set=utf8mb4
default-collation=utf8mb4_0900_ai_ci
@@ -1 +1,3 @@
schema=analytics
default-character-set=utf8mb4
default-collation=utf8mb4_0900_ai_ci
@@ -1 +1,3 @@
schema=product
default-character-set=utf8mb4
default-collation=utf8mb4_0900_ai_ci
@@ -1 +1,3 @@
schema=analytics
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=product
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=analytics
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=product
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=analytics
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=product
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=product
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=analytics
default-character-set=latin1
default-collation=latin1_swedish_ci
@@ -1 +1,3 @@
schema=product
default-character-set=latin1
default-collation=latin1_swedish_ci

0 comments on commit fe2caef

Please sign in to comment.