Permalink
Browse files

Extend ignore-schema to pull, lint, push/diff; other lint improvement

Previously, the ignore-schema option only prevented creation of matching
schema dir names in `skeema init` and `skeema pull --new-schemas`. However,
if a dir already existed, it would still be used by other commands, preventing
conditional use of ignore-schema based on environment in .skeema files.

This commit extends ignore-schema logic to all relevant commands. See #55 for
more background.

In the process of fixing that issue, this commit also makes a minor
improvement to `skeema lint` when using workspace=docker: as long as a flavor
is correctly set, `skeema lint` no longer needs to communicate with other DBs
besides the Dockerized instance. Previously, `skeema lint` would connect to
live databases unnecessarily, not actually using the connection for anything
if the flavor was already supplied in .skeema.
  • Loading branch information...
evanelias committed Jan 9, 2019
1 parent a8180e9 commit 212720e57738f85fac201eabc5eddd06783fa35a
Showing with 93 additions and 58 deletions.
  1. +1 −1 NOTICE
  2. +1 −1 README.md
  3. +1 −1 cmd_init.go
  4. +28 −4 cmd_lint.go
  5. +1 −1 cmd_pull.go
  6. +3 −3 doc/options.md
  7. +0 −2 doc/requirements.md
  8. +30 −27 fs/dir.go
  9. +28 −18 skeema_cmd_test.go
2 NOTICE
@@ -1,4 +1,4 @@
Copyright 2018 Skeema LLC
Copyright 2019 Skeema LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -63,7 +63,7 @@ Additional [contributions](https://github.com/skeema/skeema/graphs/contributors)

## License

**Copyright 2018 Skeema LLC**
**Copyright 2019 Skeema LLC**

```text
Licensed under the Apache License, Version 2.0 (the "License");
@@ -181,7 +181,7 @@ func PopulateSchemaDir(s *tengo.Schema, parentDir *fs.Dir, makeSubdir bool) erro
if ignoreSchema, err := parentDir.Config.GetRegexp("ignore-schema"); err != nil {
return NewExitValue(CodeBadConfig, err.Error())
} else if ignoreSchema != nil && ignoreSchema.MatchString(s.Name) {
log.Warnf("Skipping schema %s because of ignore-schema='%s'", s.Name, ignoreSchema)
log.Debugf("Skipping schema %s because ignore-schema='%s'", s.Name, ignoreSchema)
return nil
}

@@ -8,6 +8,7 @@ import (
"github.com/skeema/mybase"
"github.com/skeema/skeema/fs"
"github.com/skeema/skeema/workspace"
"github.com/skeema/tengo"
)

func init() {
@@ -82,19 +83,42 @@ func lintWalker(dir *fs.Dir, lc *lintCounters, maxDepth int) error {
return NewExitValue(CodeBadConfig, err.Error())
}

inst, err := dir.FirstInstance()
if err != nil {
return err
// Connect to first defined instance, unless configured to use local Docker
var inst *tengo.Instance
if wsType, _ := dir.Config.GetEnum("workspace", "temp-schema", "docker"); wsType != "docker" || !dir.Config.Changed("flavor") {
if inst, err = dir.FirstInstance(); err != nil {
return err
}
}

opts, err := workspace.OptionsForDir(dir, inst)
if err != nil {
return NewExitValue(CodeBadConfig, err.Error())
}

for _, logicalSchema := range dir.LogicalSchemas {
// ignore-schema is handled relatively simplistically here: skip dir entirely
// if any literal schema name matches the pattern, but don't bother
// interpretting schema=`shellout` or schema=*, which require an instance.
ignoreSchema, err := dir.Config.GetRegexp("ignore-schema")
if err != nil {
return NewExitValue(CodeBadConfig, err.Error())
} else if ignoreSchema != nil {
var foundIgnoredName bool
for _, schemaName := range dir.Config.GetSlice("schema", ',', true) {
if ignoreSchema.MatchString(schemaName) {
foundIgnoredName = true
}
}
if foundIgnoredName {
log.Warnf("Skipping schema in %s because ignore-schema='%s'", dir.Path, ignoreSchema)
break
}
}

schema, statementErrors, err := workspace.ExecLogicalSchema(logicalSchema, opts)
if err != nil {
log.Warnf("Skipping schema %s in %s due to error: %s", logicalSchema.Name, dir.Path, err)
log.Errorf("Skipping schema in %s due to error: %s", dir.Path, err)
lc.errCount++
continue
}
@@ -87,7 +87,7 @@ func pullWalker(dir *fs.Dir, maxDepth int) (handledSchemaNames []string, skipCou
return nil, skipCount, fmt.Errorf("%s: Unable to fetch schema names mapped by this dir: %s", dir, err)
}
if len(schemaNames) == 0 {
log.Warnf("Ignoring directory %s -- did not map to any schema names\n", dir)
log.Warnf("Ignoring directory %s -- did not map to any schema names for environment \"%s\"\n", dir, dir.Config.Get("environment"))
continue
}
handledSchemaNames = append(handledSchemaNames, schemaNames...)
@@ -463,9 +463,7 @@ The value of this option must be a valid regex, and should not be wrapped in del

When supplied on the command-line to `skeema init`, the value will be persisted into the auto-generated .skeema option file, so that subsequent commands continue to ignore the corresponding schema names.

This option primarily only affects the initial creation of a schema directory by `skeema init` or `skeema pull`. Once a schema directory is *already present*, it will be used by other commands, regardless of [ignore-schema](#ignore-schema).

Aside from the impact on schema directory creation, the only other impact of this option on other commands is to exclude specific schemas from directories configured using [schema=*](#schema), a somewhat rare sharding use-case.
Once configured, this option affects all Skeema commands, effectively acting as a filter against the [schema](#schema) option. The documentation for the [schema](#schema) option describes some potential sharding use-cases.

### ignore-table
Commands | *all*
@@ -608,6 +606,8 @@ Some sharded environments need more flexibility -- for example, where some schem
* `{DIRNAME}` -- The base name (last path element) of the directory being processed. May be useful as a key in a service discovery lookup.
* `{DIRPATH}` -- The full (absolute) path of the directory being processed.

Regardless of which form of the [schema](#schema) option is used, the [ignore-schema](#ignore-schema) option is applied as a regex "filter" against it, potentially removing some of the listed schema names based on the configuration.

### socket

Commands | *all*
@@ -14,8 +14,6 @@ Some MySQL features -- such as partitioned tables, fulltext indexes, and generat

Skeema is not currently intended for use on multi-master replication topologies, including Galera, InnoDB Cluster, and traditional active-active master-master configurations. It also has not yet been evaluated on Amazon Aurora.

As of August 2018, support for MySQL 8.0 is still quite new and should be considered experimental. Please [file an issue](https://github.com/skeema/skeema/issues/new) if you encounter anything unexpected.

### Privileges

The easiest way to run Skeema is with a user having SUPER privileges in MySQL. However, this isn't always practical or possible.
@@ -246,9 +246,11 @@ func (dir *Dir) FirstInstance() (*tengo.Instance, error) {

// SchemaNames interprets the value of the dir's "schema" option, returning one
// or more schema names that the statements in dir's *.sql files will be applied
// to, in cases where no schema name is explicitly specified.
// to, in cases where no schema name is explicitly specified in SQL statements.
// If the ignore-schema option is set, it will filter out matching results from
// the returned slice.
// An instance must be supplied since the value may be instance-specific.
func (dir *Dir) SchemaNames(instance *tengo.Instance) ([]string, error) {
func (dir *Dir) SchemaNames(instance *tengo.Instance) (names []string, err error) {
// If no schema defined in this dir (meaning this dir's .skeema, as well as
// parent dirs' .skeema, global option files, or command-line) for the current
// environment, then nothing to do
@@ -269,41 +271,42 @@ func (dir *Dir) SchemaNames(instance *tengo.Instance) ([]string, error) {
"DIRPATH": dir.Path,
}
shellOut, err := util.NewInterpolatedShellOut(schemaValue, variables)
if err == nil {
names, err = shellOut.RunCaptureSplit()
}
if err != nil {
return nil, err
}
return shellOut.RunCaptureSplit()
}

if strings.ContainsAny(schemaValue, ",") {
return dir.Config.GetSlice("schema", ',', true), nil
}

if schemaValue == "*" {
} else if schemaValue == "*" {
// This automatically already filters out information_schema, performance_schema, sys, test, mysql
schemaNames, err := instance.SchemaNames()
if err != nil {
if names, err = instance.SchemaNames(); err != nil {
return nil, err
}
// Remove ignored schemas
if ignoreSchema, err := dir.Config.GetRegexp("ignore-schema"); err != nil {
return nil, err
} else if ignoreSchema != nil {
keepNames := make([]string, 0, len(schemaNames))
for _, name := range schemaNames {
if !ignoreSchema.MatchString(name) {
keepNames = append(keepNames, name)
}
// Schema name list must be sorted so that TargetsForDir with
// firstOnly==true consistently grabs the alphabetically first schema. (Only
// relevant here since in all other cases, we use the order specified by the
// user in config.)
sort.Strings(names)
} else {
names = dir.Config.GetSlice("schema", ',', true)
}

// Remove ignored schemas
if ignoreSchema, err := dir.Config.GetRegexp("ignore-schema"); err != nil {
return nil, err
} else if ignoreSchema != nil {
keepNames := make([]string, 0, len(names))
for _, name := range names {
if ignoreSchema.MatchString(name) {
log.Debugf("Skipping schema %s because ignore-schema='%s'", name, ignoreSchema)
} else {
keepNames = append(keepNames, name)
}
schemaNames = keepNames
}
// Schema name list must be sorted so that TargetsForDir with
// firstOnly==true consistently grabs the alphabetically first schema
sort.Strings(schemaNames)
return schemaNames, nil
names = keepNames
}

return []string{schemaValue}, nil
return names, nil
}

// HasSchema returns true if this dir maps to at least one schema, either by
@@ -667,16 +667,23 @@ func (s SkeemaIntegrationSuite) TestIgnoreOptions(t *testing.T) {
cfg := s.handleCommand(t, CodeSuccess, ".", "skeema init --dir mydb -h %s -P %d --ignore-schema='^archives$' --ignore-table='^_'", s.d.Instance.Host, s.d.Instance.Port)
s.verifyFiles(t, cfg, "../golden/ignore")

// pull: nothing should be updated due to ignore options
// pull: nothing should be updated due to ignore options. Ditto even if we add
// a dir with schema name corresponding to ignored schema.
cfg = s.handleCommand(t, CodeSuccess, ".", "skeema pull")
s.verifyFiles(t, cfg, "../golden/ignore")
fs.WriteTestFile(t, "mydb/archives/.skeema", "schema=archives")
s.handleCommand(t, CodeSuccess, ".", "skeema pull")
if _, err := os.Stat("mydb/archives/foo.sql"); err == nil {
t.Error("ignore-options not affecting `skeema pull` as expected")
}

// diff/push: no differences. This should still be the case even if we add a
// file corresponding to an ignored table, with a different definition than
// the db has.
s.handleCommand(t, CodeSuccess, ".", "skeema diff")
fs.WriteTestFile(t, "mydb/product/_widgets.sql", "CREATE TABLE _widgets (id int) ENGINE=InnoDB;\n")
fs.WriteTestFile(t, "mydb/analytics/_newtable.sql", "CREATE TABLE _newtable (id int) ENGINE=InnoDB;\n")
fs.WriteTestFile(t, "mydb/archives/bar.sql", "CREATE TABLE bar (id int) ENGINE=InnoDB;\n")
s.handleCommand(t, CodeSuccess, ".", "skeema diff")

// pull should also ignore that file corresponding to an ignored table
@@ -693,16 +700,21 @@ func (s SkeemaIntegrationSuite) TestIgnoreOptions(t *testing.T) {
contents := fs.ReadTestFile(t, "mydb/analytics/_trending.sql")
newContents := strings.Replace(contents, "`", "", -1)
fs.WriteTestFile(t, "mydb/analytics/_trending.sql", newContents)
fs.WriteTestFile(t, "mydb/analytics/_hmm.sql", "lolololol no valid sql here")
fs.WriteTestFile(t, "mydb/analytics/_hmm.sql", "CREATE TABLE _hmm uhoh this is not valid;\n")
fs.WriteTestFile(t, "mydb/archives/bar.sql", "CREATE TABLE bar uhoh this is not valid;\n")
s.handleCommand(t, CodeSuccess, ".", "skeema lint")
if fs.ReadTestFile(t, "mydb/analytics/_trending.sql") != newContents {
t.Error("Expected `skeema lint` to ignore mydb/analytics/_trending.sql, but it did not")
}

// pull, lint, init: invalid regexes should error
// push, pull, lint, init: invalid regexes should error. Error is CodeBadConfig
// except for cases of invalid ignore-schema being hit in fs.Dir.SchemaNames().
s.handleCommand(t, CodeBadConfig, ".", "skeema lint --ignore-table='+'")
s.handleCommand(t, CodeBadConfig, ".", "skeema lint --ignore-schema='+'")
s.handleCommand(t, CodeBadConfig, ".", "skeema pull --ignore-table='+'")
s.handleCommand(t, CodeBadConfig, ".", "skeema pull --ignore-schema='+'")
s.handleCommand(t, CodeFatalError, ".", "skeema pull --ignore-schema='+'")
s.handleCommand(t, CodeBadConfig, ".", "skeema push --ignore-table='+'")
s.handleCommand(t, CodeFatalError, ".", "skeema push --ignore-schema='+'")
s.handleCommand(t, CodeBadConfig, ".", "skeema init --dir badre1 -h %s -P %d --ignore-schema='+'", s.d.Instance.Host, s.d.Instance.Port)
s.handleCommand(t, CodeBadConfig, ".", "skeema init --dir badre2 -h %s -P %d --ignore-table='+'", s.d.Instance.Host, s.d.Instance.Port)
}
@@ -839,16 +851,16 @@ func (s SkeemaIntegrationSuite) TestShardedSchemas(t *testing.T) {

// Make product dir now map to 3 schemas: product, product2, product3
contents := fs.ReadTestFile(t, "mydb/product/.skeema")
contents = strings.Replace(contents, "schema=product", "schema=product,product2,product3", 1)
contents = strings.Replace(contents, "schema=product", "schema=product,product2,product3,product4", 1)
fs.WriteTestFile(t, "mydb/product/.skeema", contents)

// push should now create product2 and product3
s.handleCommand(t, CodeSuccess, ".", "skeema push")
// push that ignores 4$ should now create product2 and product3
s.handleCommand(t, CodeSuccess, ".", "skeema push --ignore-schema=4$")
s.assertExists(t, "product2", "", "")
s.assertExists(t, "product3", "posts", "")

// diff should be clear after
s.handleCommand(t, CodeSuccess, ".", "skeema diff")
s.handleCommand(t, CodeSuccess, ".", "skeema diff --ignore-schema=4$")

// pull should not create separate dirs for the new schemas or mess with
// the .skeema file
@@ -868,7 +880,7 @@ func (s SkeemaIntegrationSuite) TestShardedSchemas(t *testing.T) {
// product schema or to the unsharded analytics schema
s.dbExec(t, "product", "ALTER TABLE comments ADD COLUMN `approved` tinyint(1) unsigned NOT NULL")
s.dbExec(t, "analytics", "ALTER TABLE activity ADD COLUMN `rolled_up` tinyint(1) unsigned NOT NULL")
s.handleCommand(t, CodeSuccess, ".", "skeema pull")
s.handleCommand(t, CodeSuccess, ".", "skeema pull --ignore-schema=4$")
sfContents := fs.ReadTestFile(t, "mydb/product/comments.sql")
if !strings.Contains(sfContents, "`approved` tinyint(1) unsigned") {
t.Error("Pull did not update mydb/product/comments.sql as expected")
@@ -880,20 +892,20 @@ func (s SkeemaIntegrationSuite) TestShardedSchemas(t *testing.T) {

// push should re-apply the changes to the other 2 product shards; diff
// should be clean after
s.handleCommand(t, CodeSuccess, ".", "skeema push")
s.handleCommand(t, CodeSuccess, ".", "skeema push --ignore-schema=4$")
s.assertExists(t, "product2", "comments", "approved")
s.assertExists(t, "product3", "comments", "approved")
s.handleCommand(t, CodeSuccess, ".", "skeema diff")
s.handleCommand(t, CodeSuccess, ".", "skeema diff --ignore-schema=4$")

// schema shellouts should also work properly. First get rid of product schema
// manually (since push won't ever drop a db) and then push should create
// product1 as a new schema.
contents = strings.Replace(contents, "schema=product,product2,product3", "schema=`/usr/bin/printf 'product1 product2 product3'`", 1)
contents = strings.Replace(contents, "schema=product,product2,product3,product4", "schema=`/usr/bin/printf 'product1 product2 product3 product4'`", 1)
fs.WriteTestFile(t, "mydb/product/.skeema", contents)
s.dbExec(t, "", "DROP DATABASE product")
s.handleCommand(t, CodeSuccess, ".", "skeema push")
s.handleCommand(t, CodeSuccess, ".", "skeema push --ignore-schema=4$")
s.assertExists(t, "product1", "posts", "")
s.handleCommand(t, CodeSuccess, ".", "skeema diff")
s.handleCommand(t, CodeSuccess, ".", "skeema diff --ignore-schema=4$")
s.handleCommand(t, CodeSuccess, ".", "skeema pull")
assertDirMissing("mydb/product1") // dir is still called mydb/product
assertDirMissing("mydb/product2")
@@ -907,7 +919,7 @@ func (s SkeemaIntegrationSuite) TestShardedSchemas(t *testing.T) {
if err := os.RemoveAll("mydb/analytics"); err != nil {
t.Fatalf("Unable to delete mydb/analytics/: %s", err)
}
contents = strings.Replace(contents, "schema=`/usr/bin/printf 'product1 product2 product3'`", "schema=*", 1)
contents = strings.Replace(contents, "schema=`/usr/bin/printf 'product1 product2 product3 product4'`", "schema=*", 1)
fs.WriteTestFile(t, "mydb/product/.skeema", contents)
s.handleCommand(t, CodeSuccess, ".", "skeema push --allow-unsafe")
s.assertExists(t, "product1", "posts", "")
@@ -927,10 +939,8 @@ func (s SkeemaIntegrationSuite) TestShardedSchemas(t *testing.T) {
s.handleCommand(t, CodeSuccess, ".", "skeema diff")

// Test combination of ignore-schema and schema=*
contents = strings.Replace(contents, "schema=*", "schema=*\nignore-schema=2$", 1)
fs.WriteTestFile(t, "mydb/product/.skeema", contents)
fs.WriteTestFile(t, "mydb/product/foo2.sql", "CREATE TABLE `foo2` (id int);\n")
s.handleCommand(t, CodeSuccess, ".", "skeema push")
s.handleCommand(t, CodeSuccess, ".", "skeema push --ignore-schema=2$")
s.assertExists(t, "product1", "foo2", "")
s.assertMissing(t, "product2", "foo2", "")
s.assertExists(t, "product3", "foo2", "")

0 comments on commit 212720e

Please sign in to comment.