Permalink
Browse files

Simplify optional service discovery configuration

Previously, Skeema interpreted backtick-wrapped host values as external commands
to execute. This behavior was difficult to configure, as it relied on using
directory names as lookup keys in the service discovery command-line.

This commit moves that logic to a new option, host-wrapper. With that option
configured, the host option can now be used as a lookup key to be supplied
command-line. This permits host-wrapper to be set in a single global location,
and then host can still be set on individual directories as desired. This also
permits `skeema init` and `skeema add-environment` to pass their supplied --host
values to the configured service discovery client as well.

This commit also removes old command-line placeholder variables HOSTDIR,
SCHEMADIR, and DIRPARENT as they are no longer needed and were overly complex.
  • Loading branch information...
evanelias committed Jan 21, 2017
1 parent 96bc3f4 commit 408a90ba10e61170662a845bcaf99b804d297672
Showing with 89 additions and 94 deletions.
  1. +7 −3 configutils.go
  2. +6 −9 dir.go
  3. +8 −8 dir_test.go
  4. +3 −3 doc/config.md
  5. +4 −4 doc/faq.md
  6. +53 −44 doc/options.md
  7. +7 −22 shellout.go
  8. +1 −1 shellout_test.go
View
@@ -21,16 +21,20 @@ import (
// AddGlobalOptions adds Skeema global options to the supplied mycli.Command.
// Typically cmd should be the top-level Command / Command Suite.
func AddGlobalOptions(cmd *mycli.Command) {
// Options typically only found in .skeema files -- all hidden by default
cmd.AddOption(mycli.StringOption("host", 0, "", "Database hostname or IP address").Hidden())
cmd.AddOption(mycli.StringOption("port", 0, "3306", "Port to use for database host").Hidden())
cmd.AddOption(mycli.StringOption("socket", 'S', "/tmp/mysql.sock", "Absolute path to Unix socket file used if host is localhost").Hidden())
cmd.AddOption(mycli.StringOption("schema", 0, "", "Database schema name").Hidden())
cmd.AddOption(mycli.StringOption("default-character-set", 0, "", "Schema-level default character set").Hidden())
cmd.AddOption(mycli.StringOption("default-collation", 0, "", "Schema-level default collation").Hidden())
// Visible global options
cmd.AddOption(mycli.StringOption("user", 'u', "root", "Username to connect to database host"))
cmd.AddOption(mycli.StringOption("password", 'p', "<no password>", "Password for database user; supply with no value to prompt").ValueOptional())
cmd.AddOption(mycli.StringOption("schema", 0, "", "Database schema name").Hidden())
cmd.AddOption(mycli.StringOption("host-wrapper", 'H', "", "External bin to shell out to for host lookup; see manual for template vars"))
cmd.AddOption(mycli.StringOption("temp-schema", 't', "_skeema_tmp", "Name of temporary schema for intermediate operations, created and dropped each run unless --reuse-temp-schema"))
cmd.AddOption(mycli.StringOption("connect-options", 'o', "", "Comma-separated session options to set upon connecting to each database instance"))
cmd.AddOption(mycli.StringOption("default-character-set", 0, "", "Schema-level default character set").Hidden())
cmd.AddOption(mycli.StringOption("default-collation", 0, "", "Schema-level default collation").Hidden())
cmd.AddOption(mycli.BoolOption("reuse-temp-schema", 0, false, "Do not drop temp-schema when done"))
cmd.AddOption(mycli.BoolOption("debug", 0, false, "Enable debug logging"))
}
View
15 dir.go
@@ -165,23 +165,20 @@ func (dir *Dir) Instances() ([]*tengo.Instance, error) {
socketValue := dir.Config.Get("socket")
socketWasSupplied := dir.Config.Supplied("socket")
// Interpret the host value: it may be a single literal hostname, or it may be
// a backtick-wrapped shellout.
hostValue := dir.Config.Get("host") // Get strips quotes (including backticks) from fully quoted-wrapped values
rawHostValue := dir.Config.GetRaw("host") // GetRaw does not strip quotes
// Interpret the host value: if host-wrapper is set, use it to interpret the
// host list; otherwise assume host is a comma-separated list of literal
// hostnames.
var hosts []string
if rawHostValue != hostValue && rawHostValue[0] == '`' { // no need to check len, the Changed check above already tells us host != ""
s, err := NewInterpolatedShellOut(hostValue, dir, nil)
if dir.Config.Changed("host-wrapper") {
s, err := NewInterpolatedShellOut(dir.Config.Get("host-wrapper"), dir, nil)
if err != nil {
return nil, err
}
if hosts, err = s.RunCaptureSplit(); err != nil {
return nil, err
}
} else if strings.ContainsAny(hostValue, ",") {
hosts = dir.Config.GetSlice("host", ',', true)
} else {
hosts = []string{hostValue}
hosts = dir.Config.GetSlice("host", ',', true)
}
// For each hostname, construct a DSN and use it to create an Instance
View
@@ -82,15 +82,15 @@ func TestInstances(t *testing.T) {
assertInstances(map[string]string{"host": "some.db.host", "connect-options": ","}, true)
assertInstances(map[string]string{"host": "some.db.host:3306", "port": "3307"}, true)
assertInstances(map[string]string{"host": "@@@@@"}, true)
assertInstances(map[string]string{"host": "`echo {INVALID_VAR}`"}, true)
assertInstances(map[string]string{"host-wrapper": "`echo {INVALID_VAR}`", "host": "irrelevant"}, true)
// dynamic hosts via command execution
assertInstances(map[string]string{"host": "`/usr/bin/printf 'some.db.host'`"}, false, "some.db.host:3306")
assertInstances(map[string]string{"host": "`/usr/bin/printf 'some.db.host\n'`"}, false, "some.db.host:3306")
assertInstances(map[string]string{"host": "`/usr/bin/printf 'some.db.host\nother.db.host'`", "port": "3333"}, false, "some.db.host:3333", "other.db.host:3333")
assertInstances(map[string]string{"host": "`/usr/bin/printf 'some.db.host\tother.db.host:3316'`", "port": "3316"}, false, "some.db.host:3316", "other.db.host:3316")
assertInstances(map[string]string{"host": "`/usr/bin/printf 'localhost,remote.host:3307,other.host'`", "socket": "/var/lib/mysql/mysql.sock"}, false, "localhost:/var/lib/mysql/mysql.sock", "remote.host:3307", "other.host:3306")
assertInstances(map[string]string{"host": "`/bin/echo -n`"}, false)
// dynamic hosts via host-wrapper command execution
assertInstances(map[string]string{"host-wrapper": "/usr/bin/printf '{HOST}:3306'", "host": "some.db.host"}, false, "some.db.host:3306")
assertInstances(map[string]string{"host-wrapper": "`/usr/bin/printf '{HOST}\n'`", "host": "some.db.host:3306"}, false, "some.db.host:3306")
assertInstances(map[string]string{"host-wrapper": "/usr/bin/printf 'some.db.host\nother.db.host'", "host": "ignored", "port": "3333"}, false, "some.db.host:3333", "other.db.host:3333")
assertInstances(map[string]string{"host-wrapper": "/usr/bin/printf 'some.db.host\tother.db.host:3316'", "host": "ignored", "port": "3316"}, false, "some.db.host:3316", "other.db.host:3316")
assertInstances(map[string]string{"host-wrapper": "/usr/bin/printf 'localhost,remote.host:3307,other.host'", "host": "ignored", "socket": "/var/lib/mysql/mysql.sock"}, false, "localhost:/var/lib/mysql/mysql.sock", "remote.host:3307", "other.host:3306")
assertInstances(map[string]string{"host-wrapper": "/bin/echo -n", "host": "ignored"}, false)
}
func TestInstanceDefaultParams(t *testing.T) {
View
@@ -126,18 +126,18 @@ Most other commands (`skeema diff`, `skeema push`, `skeema pull`, `skeema lint`)
### Options with variable interpolation
Some string-type options, such as [alter-wrapper](options.md#alter-wrapper), are always interpreted as external commands to execute. A few other string-type options, such as [host](options.md#host), are optionally interpreted as external commands if the value is wrapped in backticks.
Some string-type options, such as [alter-wrapper](options.md#alter-wrapper), are always interpreted as external commands to execute. A few other string-type options, such as [schema](options.md#schema), are optionally interpreted as external commands only if the entire option value is wrapped in backticks.
In either case, the external command-line supports interpolation of variable placeholders, which appear in all-caps and are wrapped in braces like `{VARNAME}`. For example, this line may appear in a .skeema file to configure use of pt-online-schema-change:
```ini
alter-wrapper=/usr/local/bin/pt-online-schema-change --alter {CLAUSES} D={SCHEMA},t={TABLE},h={HOST},P={PORT},u={USER},p={PASSWORDX}
```
Or this line might be used in a .skeema file to configure service discovery for dynamically mapping the directory to database instances, based on the environment name and directory name:
Or this line might be used in a .skeema file to configure service discovery via [host-wrapper](options.md#host-wrapper), to dynamically map [host](options.md#host) values to database instances, instead of using the host value literally as an address:
```ini
host=`/path/to/service_discovery_lookup.sh {ENVIRONMENT} {DIRPARENT}`
host-wrapper=/path/to/service_discovery_lookup.sh /databases/{ENVIRONMENT}/{HOST}
```
The placeholders are automatically replaced with the correct values for the current operation. Each option lists what variables it supports.
View
@@ -91,12 +91,12 @@ Note that these ALTERs generally aren't replication-friendly due to lag they cre
### How do I configure Skeema to use service discovery?
There are several possibilities here, all based on how the [host option](options.md#host) is configured:
There are several possibilities here, all based on how the [host](options.md#host) and [host-wrapper](options.md#host-wrapper) options are configured:
* DNS: This works if you can provide a consistently up-to-date domain name for the master of each pool. It isn't friendly towards sharded environments though, nor is it a good solution if nonstandard port numbers are in use. (Skeema does not yet support SRV record lookups.)
* DNS: [host](options.md#host) set to a domain name, and [host-wrapper](options.md#host-wrapper) left blank. This works if you can provide a consistently up-to-date domain name for the master of each pool. It isn't friendly towards sharded environments though, nor is it a good solution if nonstandard port numbers are in use. (Skeema does not yet support SRV record lookups.)
* External command shellout: by setting [host](options.md#host) to a backtick-wrapped external command line, you can configure Skeema to obtain hosts (and optionally ports) dynamically from the output of any arbitrary script. This permits you to interface with any service discovery client, to do lookups like "return the master of pool foo" or "return all shard masters for sharded pool xyz".
* External command shellout: configuring [host-wrapper](options.md#host-wrapper) to shell out to a service discovery client. In this configuration, rather than [host](options.md#host) being set to a literal address, it should be a lookup key to pass to service discovery. The [host-wrapper](options.md#host-wrapper) command-line can then use the `{HOST}` placeholder variable to obtain each directory's lookup key, such as `host-wrapper=/path/to/service_discovery_lookup.sh /databases/{ENVIRONMENT}/{HOST}`. The executed script should be capable of doing lookups such as "return the master of pool foo" or "return all shard masters for sharded pool xyz".
* Configuration management: You could use a system like Chef or Puppet to rewrite directories' .skeema config files periodically, ensuring that an up-to-date master IP is listed in each file.
* Configuration management: You could use a system like Chef or Puppet to rewrite directories' .skeema config files periodically, ensuring that an up-to-date master IP is listed for [host](options.md#host) in each file.
Simpler integration with etcd, Consul, and ZooKeeper is planned for future releases.
Oops, something went wrong.

0 comments on commit 408a90b

Please sign in to comment.