From 0dfa3396cea8cf2f625a8f41a06615a9687fce76 Mon Sep 17 00:00:00 2001 From: Eduardo Pereira Date: Sat, 30 Jun 2018 23:43:12 +0100 Subject: [PATCH 1/5] first release (#2) * Add info about releases * add new configuration flag for valid input file extensions * add go dep for dependency management * add barebones Config test + update dependencies with gomega * create interface for configs to make it testable * update dependencies * add first tests to Configurations * add getter for new config param * new logic on valid file extensions * update documentation * add some files for testing * make Consul ACL flag optional * only set Consul ACL header when flag is provided * trim any dots on file extensions * minor documentation fixes --- .gitignore | 5 +- Gopkg.lock | 285 ++++++++++++++++++++++ Gopkg.toml | 54 ++++ README.md | 56 +++-- configuration/config.go | 120 +++++---- configuration/config_test.go | 117 +++++++++ configuration/flags.go | 45 ++++ exporter/directory.go | 13 +- importer/consul.go | 4 +- importer/helper.go | 6 +- interfaces/configurations.go | 25 ++ main.go | 4 +- tests/mocks/.gitkeep | 0 tests/test-repo/dev/app1/application.ini | 6 + tests/test-repo/dev/app1/config.json | 4 + tests/test-repo/dev/app1/db-pass.txt | 1 + tests/test-repo/dev/app1/gateways.yaml | 8 + tests/test-repo/prod/app1/application.ini | 6 + tests/test-repo/prod/app1/config.json | 4 + tests/test-repo/prod/app1/db-pass.txt | 1 + tests/test-repo/prod/app1/gateways.yaml | 8 + tests/test-repo/stg/app1/application.ini | 6 + tests/test-repo/stg/app1/config.json | 4 + tests/test-repo/stg/app1/db-pass.txt | 1 + tests/test-repo/stg/app1/gateways.yaml | 8 + tests/test-secrets-file-fail.json | 3 + tests/test-secrets-file-success.json | 3 + 27 files changed, 711 insertions(+), 86 deletions(-) create mode 100644 Gopkg.lock create mode 100644 Gopkg.toml create mode 100644 configuration/config_test.go create mode 100644 configuration/flags.go create mode 100644 interfaces/configurations.go create mode 100644 tests/mocks/.gitkeep create mode 100644 tests/test-repo/dev/app1/application.ini create mode 100644 tests/test-repo/dev/app1/config.json create mode 100644 tests/test-repo/dev/app1/db-pass.txt create mode 100644 tests/test-repo/dev/app1/gateways.yaml create mode 100644 tests/test-repo/prod/app1/application.ini create mode 100644 tests/test-repo/prod/app1/config.json create mode 100644 tests/test-repo/prod/app1/db-pass.txt create mode 100644 tests/test-repo/prod/app1/gateways.yaml create mode 100644 tests/test-repo/stg/app1/application.ini create mode 100644 tests/test-repo/stg/app1/config.json create mode 100644 tests/test-repo/stg/app1/db-pass.txt create mode 100644 tests/test-repo/stg/app1/gateways.yaml create mode 100644 tests/test-secrets-file-fail.json create mode 100644 tests/test-secrets-file-success.json diff --git a/.gitignore b/.gitignore index 723ef36..2b9bcee 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ -.idea \ No newline at end of file +.idea +vendor +tests/mocks/* +!tests/mocks/.gitkeep \ No newline at end of file diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..4596b7c --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,285 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/cbroglie/mustache" + packages = ["."] + revision = "eb931a9d20042e51d8a3a9ad5a9ba9bc8282c564" + version = "v1.0.1" + +[[projects]] + name = "github.com/davecgh/go-spew" + packages = ["spew"] + revision = "346938d642f2ec3594ed81d874461961cd0faa76" + version = "v1.1.0" + +[[projects]] + name = "github.com/emirpasic/gods" + packages = [ + "containers", + "lists", + "lists/arraylist", + "trees", + "trees/binaryheap", + "utils" + ] + revision = "f6c17b524822278a87e3b3bd809fec33b51f5b46" + version = "v1.9.0" + +[[projects]] + name = "github.com/gorilla/context" + packages = ["."] + revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42" + version = "v1.1.1" + +[[projects]] + name = "github.com/gorilla/mux" + packages = ["."] + revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf" + version = "v1.6.2" + +[[projects]] + branch = "master" + name = "github.com/jbenet/go-context" + packages = ["io"] + revision = "d14ea06fba99483203c19d92cfcd13ebe73135f4" + +[[projects]] + name = "github.com/kevinburke/ssh_config" + packages = ["."] + revision = "9fc7bb800b555d63157c65a904c86a2cc7b4e795" + version = "0.4" + +[[projects]] + name = "github.com/mattn/go-runewidth" + packages = ["."] + revision = "9e777a8366cce605130a531d2cd6363d07ad7317" + version = "v0.0.2" + +[[projects]] + branch = "master" + name = "github.com/mitchellh/go-homedir" + packages = ["."] + revision = "3864e76763d94a6df2f9960b16a20a33da9f9a66" + +[[projects]] + branch = "master" + name = "github.com/olekukonko/tablewriter" + packages = ["."] + revision = "d4647c9c7a84d847478d890b816b7d8b62b0b279" + +[[projects]] + name = "github.com/onsi/gomega" + packages = [ + ".", + "format", + "internal/assertion", + "internal/asyncassertion", + "internal/oraclematcher", + "internal/testingtsupport", + "matchers", + "matchers/support/goraph/bipartitegraph", + "matchers/support/goraph/edge", + "matchers/support/goraph/node", + "matchers/support/goraph/util", + "types" + ] + revision = "62bff4df71bdbc266561a0caee19f0594b17c240" + version = "v1.4.0" + +[[projects]] + name = "github.com/pelletier/go-buffruneio" + packages = ["."] + revision = "c37440a7cf42ac63b919c752ca73a85067e05992" + version = "v0.2.0" + +[[projects]] + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + name = "github.com/sergi/go-diff" + packages = ["diffmatchpatch"] + revision = "1744e2970ca51c86172c8190fadad617561ed6e7" + version = "v1.0.0" + +[[projects]] + name = "github.com/src-d/gcfg" + packages = [ + ".", + "scanner", + "token", + "types" + ] + revision = "f187355171c936ac84a82793659ebb4936bc1c23" + version = "v1.3.0" + +[[projects]] + name = "github.com/stretchr/objx" + packages = ["."] + revision = "477a77ecc69700c7cdeb1fa9e129548e1c1c393c" + version = "v0.1.1" + +[[projects]] + name = "github.com/stretchr/testify" + packages = [ + ".", + "assert", + "http", + "mock" + ] + revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" + version = "v1.2.2" + +[[projects]] + branch = "master" + name = "github.com/xanzy/ssh-agent" + packages = ["."] + revision = "ba9c9e33906f58169366275e3450db66139a31a9" + +[[projects]] + branch = "master" + name = "golang.org/x/crypto" + packages = [ + "cast5", + "curve25519", + "ed25519", + "ed25519/internal/edwards25519", + "internal/chacha20", + "internal/subtle", + "openpgp", + "openpgp/armor", + "openpgp/elgamal", + "openpgp/errors", + "openpgp/packet", + "openpgp/s2k", + "poly1305", + "ssh", + "ssh/agent", + "ssh/knownhosts" + ] + revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = [ + "context", + "html", + "html/atom", + "html/charset" + ] + revision = "4cb1c02c05b0e749b0365f61ae859a8e0cfceed9" + +[[projects]] + branch = "master" + name = "golang.org/x/sys" + packages = ["windows"] + revision = "7138fd3d9dc8335c567ca206f4333fb75eb05d56" + +[[projects]] + name = "golang.org/x/text" + packages = [ + "encoding", + "encoding/charmap", + "encoding/htmlindex", + "encoding/internal", + "encoding/internal/identifier", + "encoding/japanese", + "encoding/korean", + "encoding/simplifiedchinese", + "encoding/traditionalchinese", + "encoding/unicode", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "internal/utf8internal", + "language", + "runes", + "transform", + "unicode/cldr", + "unicode/norm" + ] + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" + +[[projects]] + name = "gopkg.in/src-d/go-billy.v4" + packages = [ + ".", + "helper/chroot", + "helper/polyfill", + "osfs", + "util" + ] + revision = "83cf655d40b15b427014d7875d10850f96edba14" + version = "v4.2.0" + +[[projects]] + name = "gopkg.in/src-d/go-git.v4" + packages = [ + ".", + "config", + "internal/revision", + "plumbing", + "plumbing/cache", + "plumbing/filemode", + "plumbing/format/config", + "plumbing/format/diff", + "plumbing/format/gitignore", + "plumbing/format/idxfile", + "plumbing/format/index", + "plumbing/format/objfile", + "plumbing/format/packfile", + "plumbing/format/pktline", + "plumbing/object", + "plumbing/protocol/packp", + "plumbing/protocol/packp/capability", + "plumbing/protocol/packp/sideband", + "plumbing/revlist", + "plumbing/storer", + "plumbing/transport", + "plumbing/transport/client", + "plumbing/transport/file", + "plumbing/transport/git", + "plumbing/transport/http", + "plumbing/transport/internal/common", + "plumbing/transport/server", + "plumbing/transport/ssh", + "storage", + "storage/filesystem", + "storage/filesystem/dotgit", + "storage/memory", + "utils/binary", + "utils/diff", + "utils/ioutil", + "utils/merkletrie", + "utils/merkletrie/filesystem", + "utils/merkletrie/index", + "utils/merkletrie/internal/frame", + "utils/merkletrie/noder" + ] + revision = "b23570073eaee3489e5e3d666f22ba5cbeb53243" + version = "v4.4.1" + +[[projects]] + name = "gopkg.in/warnings.v0" + packages = ["."] + revision = "ec4a0fea49c7b46c2aeb0b51aac55779c607e52b" + version = "v0.1.2" + +[[projects]] + name = "gopkg.in/yaml.v2" + packages = ["."] + revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" + version = "v2.2.1" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "00c6f0e183bc2a3d67185a03caa8050c26f3156dae8b25fe5c39b7f7c374797c" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..e38fea9 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,54 @@ +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + + +[[constraint]] + name = "github.com/cbroglie/mustache" + version = "1.0.1" + +[[constraint]] + name = "github.com/gorilla/mux" + version = "1.6.2" + +[[constraint]] + branch = "master" + name = "github.com/olekukonko/tablewriter" + +[[constraint]] + name = "github.com/onsi/gomega" + version = "1.4.0" + +[[constraint]] + name = "gopkg.in/src-d/go-git.v4" + version = "4.4.1" + +[prune] + go-tests = true + unused-packages = true + +[[constraint]] + name = "github.com/stretchr/testify" + version = "1.2.2" diff --git a/README.md b/README.md index 8e43524..60af51d 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,16 @@ -# Gonsul - Git to Consul tool, in GO! +# Gonsul - A Git to Consul tool, in Go! This tool serves as an entry point for the Hashicorp's Consul KV store. Not only because Consul lacks of a built in audit mechanism, but also because having configurations managed in GIT, using a gitflow or a normal -development-to-master flow is much friedly and familiar to any development team to manage configurations. +development-to-master flow is much friendly and familiar to any development team to manage configurations. + +Downloads in [releases page](https://github.com/miniclip/gonsul/releases). ## How It Processes a Repository Gonsul will (optionally) clone your repository into the filesystem. After, it will recursively parse all the files in the directory. Whenever Gonsul moves one level deep into a folder, the folder name is added as a Consul KV path part -and as soon as it finds a file (either `.json`, `.txt` or `.ini`) it will take the file name (without the extension) and -append to the Consul KV path, making it a key and the file content is added as the value. +and as soon as it finds a file (either `.json`, `.txt` or `.ini` - or any other given in parameters) it will take the +file name (without the extension) and append to the Consul KV path, making it a key and the file content is added as the +value. **Example:** Take this repository folder structure: ``` @@ -88,6 +91,7 @@ Below are all available command line flags for starting **Gonsul** --secrets-file= --allow-deletes= --poll-interval= +--input-ext= ``` Below is the full description for each individual command line flag @@ -196,7 +200,7 @@ Consul's KV store. Please provide the full URL, with scheme and port if appropri ### `--consul-acl` -> `require:` **yes** +> `require:` **no** > `example:` **`--consul-acl=youracltokenhere`** This is the Consul's access token that Gonsul will use when connecting to the cluster agent. This ACL **must** have read @@ -263,7 +267,7 @@ This file must be a valid JSON map, where the keys are the placeholders and the } ``` **Note 1:** All the replacement is done on-the-fly in memory, and apart from the original supplied `secrets.json` file, -no secrets are written to disk. +no secrets are written to disk. **Note 2:** The placeholders **should** follow the *mustache* triple curly braces `{{{FOO_DB_USER}}}`, that means *"unescaped HTML charcaters"* - basically takes the value as is. @@ -295,40 +299,38 @@ monitor Gonsul logs to detect any found errors, and react appropriately. The err This is the number of seconds you want Gonsul to wait between checks on the repository when it is running in `--strategy=POLL` mode. + +### `--input-ext` +> `require:` **no** +> `default:` **json,txt,ini** +> `example:` **`--input-ext=json,txt,ini,yaml`** + +This is the file extensions that Gonsul should consider as inputs to populate our Consul. Please set each extension +without the dot, and separate each extension with a comma. + ## Gonsul Exit Codes Whenever an error occurs, and Gonsul exits with a code other than 0, we try to return a meaningful code, such as: -### 10 -This is the most important error code. It means **Delete** operations were found and Gonsul is running without +* **10** - This is the most important error code. It means **Delete** operations were found and Gonsul is running without delete permission. This error comes with the info about the Consul KV paths that would be deleted. -### 20 -There was a problem on the initialization parameters /flags +* **20** - There was a problem on the initialization parameters /flags -### 30 -This means there was an error connecting to Consul cluster. This can ben either ACL token, wrong endpoint, network, etc. +* **30** - This means there was an error connecting to Consul cluster. This can ben either ACL token, wrong endpoint, network, etc. -### 31 -There was a problem running a Consul transaction. It basically means on operation of the transaction is corrupted for +* **31** - There was a problem running a Consul transaction. It basically means on operation of the transaction is corrupted for some reason. Try a dryrun to analyze all the operations Gonsul is trying to run. -### 40 -This is a generic error when Gonsul fails to read an HTTP response. +* **40** - This is a generic error when Gonsul fails to read an HTTP response. -### 50 -This error is thrown when Gonsul could not encode a json payload for a transaction. Check **dryrun** for what operations +* **50** - This error is thrown when Gonsul could not encode a json payload for a transaction. Check **dryrun** for what operations Gonsul is trying to run. -### 51 -This is when Gonsul could not decode a JSON payload. This can be either from a GET response from Consul, or more common +* **51** - This is when Gonsul could not decode a JSON payload. This can be either from a GET response from Consul, or more common when processing the filesystem and it found a corrupted JSON file - check your JSON files for errors. +* **60** - This occurs when Gonsul cannot clone the repository. Either because credentials are broken, or filesystem permissions. -### 60 -This occurs when Gonsul cannot clone the repository. Either because credentials are broken, or filesystem permissions. - -### 70 -This error occurs when secret replacement fails. +* **70** - This error occurs when secret replacement fails. -### 80 -This is a generic HTTP error. Run Gonsul in debug mode to look for more information regarding the error. \ No newline at end of file +* **80** - This is a generic HTTP error. Run Gonsul in debug mode to look for more information regarding the error. \ No newline at end of file diff --git a/configuration/config.go b/configuration/config.go index d73e07f..7e38f7a 100644 --- a/configuration/config.go +++ b/configuration/config.go @@ -10,6 +10,7 @@ import ( "io/ioutil" "os" "strings" + "github.com/miniclip/gonsul/interfaces" ) const StrategyDry = "DRYRUN" @@ -17,23 +18,6 @@ const StrategyOnce = "ONCE" const StrategyPoll = "POLL" const StrategyHook = "HOOK" -var logLevel = flag.String("log-level", errorutil.LogErr, fmt.Sprintf("The desired log level (%s, %s, %s)", errorutil.LogErr, errorutil.LogInfo, errorutil.LogDebug)) -var strategyFlag = flag.String("strategy", StrategyOnce, fmt.Sprintf("The Gonsul operation mode (%s, %s, %s, %s)", StrategyDry, StrategyOnce, StrategyPoll, StrategyHook)) -var repoURLFlag = flag.String("repo-url", "", "The repository URL (Full URL with scheme)") -var repoSSHKeyFlag = flag.String("repo-ssh-key", "", "The SSH private key location (Full path)") -var repoSSHUserFlag = flag.String("repo-ssh-user", "git", "The SSH user name") -var repoBranchFlag = flag.String("repo-branch", "master", "Which branch should we look at") -var repoRemoteNameFlag = flag.String("repo-remote-name", "origin", "The repository remote name") -var repoBasePathFlag = flag.String("repo-base-path", "/", "The base directory to look from inside the repo") -var repoRootDirFlag = flag.String("repo-root", "/tmp/gonsul/repo", "The path where the repo will be downloaded to") -var consulURLFlag = flag.String("consul-url", "", "(REQUIRED) The Consul URL REST API endpoint (Full URL with scheme)") -var consulACLFlag = flag.String("consul-acl", "", "(REQUIRED) The Consul ACL to use (Must have write on the KV following --consul-base path)") -var consulBasePathFlag = flag.String("consul-base-path", "", "The base KV path will be prefixed to dir path - DO NOT START NOR END WITH SLASH") -var expandJSONFlag = flag.Bool("expand-json", false, "Expand and parse JSON files as full paths?") -var secretsFile = flag.String("secrets-file", "", "A key value json file with placeholders->secrets mapping, in order to do on the fly replace") -var allowDeletesFlag = flag.Bool("allow-deletes", false, "Show Gonsul issue deletes? (If not, nothing will be done and a report on conflicting deletes will be shown)") -var pollIntervalFlag = flag.Int("poll-interval", 60, "The number of seconds for the repository polling interval") - var config *Config type Config struct { @@ -56,21 +40,29 @@ type Config struct { allowDeletes bool pollInterval int Working chan bool + validExtensions []string } -func GetConfig() (*Config, error) { - +func GetConfig(flagParser interfaces.IConfigFlags) (*Config, error) { + // Set our local error var var err error - + // Singleton check if config == nil { - config, err = buildConfig() - return config, err + // Parse our flags + flags := flagParser.Parse() + // Build our configuration + config, err = buildConfig(flags) } - return config, nil + // Return the config and error (whichever state they are) + return config, err +} + +func DestroyConfig() { + config = nil } -func buildConfig() (*Config, error) { +func buildConfig(flags interfaces.ConfigFlags) (*Config, error) { // Set some local variable and some others defaulted var secrets map[string]string @@ -78,17 +70,20 @@ func buildConfig() (*Config, error) { clone := true doSecrets := false - // Parse our command line flags - flag.Parse() - // Make sure we have the mandatory flags set - if *consulURLFlag == "" || *consulACLFlag == "" { + if *flags.ConsulURL == "" || *flags.ValidExtensions == "" { flag.PrintDefaults() return nil, errors.New("required flags not set") } + // Set our valid extensions + extensions, err := setValidExtensions(*flags.ValidExtensions) + if err != nil { + return nil, err + } + // Make sure strategy is properly given - strategy := strings.ToUpper(*strategyFlag) + strategy := strings.ToUpper(*flags.Strategy) if strategy != StrategyDry && strategy != StrategyOnce && strategy != StrategyPoll && strategy != StrategyHook { return nil, errors.New(fmt.Sprintf("strategy invalid, must be one of: %s, %s, %s, %s", StrategyDry, StrategyOnce, StrategyPoll, StrategyHook)) } @@ -96,19 +91,19 @@ func buildConfig() (*Config, error) { // Shall we use a local copy of the repository instead of cloning ourselves // This should be useful if we use Gonsul on a CI stack (such as Bamboo) // And the repo is checked out already, alleviating Gonsul work - if *repoURLFlag == "" && *repoRootDirFlag != "" { + if *flags.RepoURL == "" && *flags.RepoRootDir != "" { clone = false } // Make sure log level is properly set - errorLevel := errorutil.ErrorLevels[strings.ToUpper(*logLevel)] + errorLevel := errorutil.ErrorLevels[strings.ToUpper(*flags.LogLevel)] if errorLevel < errorutil.LogLevelErr { return nil, errors.New(fmt.Sprintf("log level invalid, must be one of: %s, %s, %s", errorutil.LogErr, errorutil.LogInfo, errorutil.LogDebug)) } // Should we build a secrets map for on-the-fly mustache replacement - if *secretsFile != "" { - secrets, err = buildSecretsMap(*secretsFile, *repoRootDirFlag) + if *flags.SecretsFile != "" { + secrets, err = buildSecretsMap(*flags.SecretsFile, *flags.RepoRootDir) if err != nil { return nil, err } @@ -116,25 +111,26 @@ func buildConfig() (*Config, error) { } return &Config{ - shouldClone: clone, - logLevel: errorLevel, - strategy: strategy, - repoUrl: *repoURLFlag, - repoSSHKey: *repoSSHKeyFlag, - repoSSHUser: *repoSSHUserFlag, - repoBranch: *repoBranchFlag, - repoRemoteName: *repoRemoteNameFlag, - repoBasePath: *repoBasePathFlag, - repoRootDir: *repoRootDirFlag, - consulURL: *consulURLFlag, - consulACL: *consulACLFlag, - consulBasePath: *consulBasePathFlag, - expandJSON: *expandJSONFlag, - doSecrets: doSecrets, - secretsMap: secrets, - allowDeletes: *allowDeletesFlag, - pollInterval: *pollIntervalFlag, - Working: make(chan bool, 1), + shouldClone: clone, + logLevel: errorLevel, + strategy: strategy, + repoUrl: *flags.RepoURL, + repoSSHKey: *flags.RepoSSHKey, + repoSSHUser: *flags.RepoSSHUser, + repoBranch: *flags.RepoBranch, + repoRemoteName: *flags.RepoRemoteName, + repoBasePath: *flags.RepoBasePath, + repoRootDir: *flags.RepoRootDir, + consulURL: *flags.ConsulURL, + consulACL: *flags.ConsulACL, + consulBasePath: *flags.ConsulBasePath, + expandJSON: *flags.ExpandJSON, + doSecrets: doSecrets, + secretsMap: secrets, + allowDeletes: *flags.AllowDeletes, + pollInterval: *flags.PollInterval, + Working: make(chan bool, 1), + validExtensions: extensions, }, nil } @@ -210,6 +206,10 @@ func (config *Config) GetPollInterval() int { return config.pollInterval } +func (config *Config) GetValidExtensions() []string { + return config.validExtensions +} + func buildSecretsMap(secretsFile string, repoRootPath string) (map[string]string, error) { var file = secretsFile if _, err := os.Stat(file); os.IsNotExist(err) { @@ -237,3 +237,19 @@ func buildSecretsMap(secretsFile string, repoRootPath string) (map[string]string return secretsMap, nil } + +func setValidExtensions(validExtensions string) ([]string, error) { + var extensionsArr []string + + // Try to explode the string + extensions := strings.Split(validExtensions, ",") + if len(extensions) < 1 { + return nil, errors.New(fmt.Sprintf("could not open get valid extensions from flag (%s). Value given: %s", "--input-ext", validExtensions)) + } + + for _, extension := range extensions { + extensionsArr = append(extensionsArr, extension) + } + + return extensionsArr, nil +} diff --git a/configuration/config_test.go b/configuration/config_test.go new file mode 100644 index 0000000..51c2e2e --- /dev/null +++ b/configuration/config_test.go @@ -0,0 +1,117 @@ +package configuration + +import ( + "testing" + . "github.com/onsi/gomega" + + "github.com/miniclip/gonsul/tests/mocks" + "github.com/miniclip/gonsul/errorutil" + "github.com/miniclip/gonsul/interfaces" + "fmt" +) + +func TestGetConfigSuccess(t *testing.T) { + RegisterTestingT(t) + + // Instantiate our mocks + flagsMock := &mocks.IConfigFlags{} + // Get our mocked flags + configFlags := getConfigFlagsFor( + errorutil.LogDebug, + StrategyOnce, + "", + "", + "", + "", + "", + "/", + "./..", + "http://consul.com", + "some-acl-1234567890-qwerty", + "", + false, + "tests/test-secrets-file-success.json", + false, + 60, + "json,txt,ini", + ) + + // Setup expectations + flagsMock.On("Parse").Return(configFlags) + + configs, err := GetConfig(flagsMock) + + Expect(flagsMock.AssertExpectations(t)).Should(BeTrue(), "Mocked method must be called") + Expect(err).To(BeNil()) + Expect(configs).To(Not(BeNil())) +} + +func TestGetConfigMultipleFail(t *testing.T) { + RegisterTestingT(t) + + // Instantiate our mocks + flagsMock := &mocks.IConfigFlags{} + // Get our mocked flags + configFlags := getMultipleWrongConfigs() + + for i, badConfigFlags := range configFlags { + // Reset our singleton config + DestroyConfig() + + // Setup expectations + flagsMock.On("Parse").Return(badConfigFlags) + + // Run our tested function injecting our mock + configs, err := GetConfig(flagsMock) + + // Assert our expectations + Expect(flagsMock.AssertExpectations(t)).Should(BeTrue(), fmt.Sprintf("Mocked method must be called (%d)", i)) + Expect(err).To(Not(BeNil()), fmt.Sprintf("Error must no be nil (%d)", i)) + Expect(configs).To(BeNil(), fmt.Sprintf("Configs must be nil (%d)", i)) + } +} + + +func getMultipleWrongConfigs() []interfaces.ConfigFlags { + return []interfaces.ConfigFlags{ + getConfigFlagsFor("WRONG_LOG_LEVEL", StrategyOnce, "", "", "", "", "", "/", "./..", "http://consul.com", "some-acl-1234567890-qwerty", "", false, "tests/test-secrets-file-success.json", false, 60, "json,txt,ini"), + getConfigFlagsFor(errorutil.LogDebug, "WRONG_STRATEGY", "", "", "", "", "", "/", "./..", "http://consul.com", "some-acl-1234567890-qwerty", "", false, "tests/test-secrets-file-success.json", false, 60, "json,txt,ini"), + getConfigFlagsFor(errorutil.LogDebug, StrategyOnce, "", "", "", "", "", "/", "./..", "", "some-acl-1234567890-qwerty", "", false, "tests/test-secrets-file-success.json", false, 60, "json,txt,ini"), + getConfigFlagsFor(errorutil.LogDebug, StrategyOnce, "", "", "", "", "", "/", "./..", "http://consul.com", "", "", false, "tests/test-secrets-file-success.json", false, 60, "json,txt,ini"), + getConfigFlagsFor(errorutil.LogDebug, StrategyOnce, "", "", "", "", "", "/", "./..", "http://consul.com", "some-acl-1234567890-qwerty", "", false, "tests/test-secrets-file-success.json", false, 60, ""), + getConfigFlagsFor(errorutil.LogDebug, StrategyOnce, "", "", "", "", "", "/", "./..", "http://consul.com", "some-acl-1234567890-qwerty", "", false, "tests/test-secrets-file-fail.json", false, 60, "json,txt,ini"), + getConfigFlagsFor(errorutil.LogDebug, StrategyOnce, "", "", "", "", "", "/", "./..", "http://consul.com", "some-acl-1234567890-qwerty", "", false, "tests/test-secrets-file-non-existent.json", false, 60, "json,txt,ini"), + } +} + + +func getConfigFlagsFor( + ll, s, ru, rsk, rsu, rb, rrn, rbp, rr, cu, ca, cbp string, + ej bool, + sf string, + ad bool, + pi int, + ie string, +) interfaces.ConfigFlags { + configFlags := interfaces.ConfigFlags{ + LogLevel: &ll, + Strategy: &s, + RepoURL: &ru, + RepoSSHKey: &rsk, + RepoSSHUser: &rsu, + RepoBranch: &rb, + RepoRemoteName: &rrn, + RepoBasePath: &rbp, + RepoRootDir: &rr, + ConsulURL: &cu, + ConsulACL: &ca, + ConsulBasePath: &cbp, + ExpandJSON: &ej, + SecretsFile: &sf, + AllowDeletes: &ad, + PollInterval: &pi, + ValidExtensions: &ie, + } + + return configFlags +} \ No newline at end of file diff --git a/configuration/flags.go b/configuration/flags.go new file mode 100644 index 0000000..800ca50 --- /dev/null +++ b/configuration/flags.go @@ -0,0 +1,45 @@ +package configuration + +import ( + "flag" + "fmt" + "github.com/miniclip/gonsul/errorutil" + "github.com/miniclip/gonsul/interfaces" +) + + + +type ConfigFlagsParser struct { + Flags interfaces.ConfigFlags +} + +func (flags *ConfigFlagsParser) Parse() interfaces.ConfigFlags { + flags.Flags.LogLevel = flag.String("log-level", errorutil.LogErr, fmt.Sprintf("The desired log level (%s, %s, %s)", errorutil.LogErr, errorutil.LogInfo, errorutil.LogDebug)) + flags.Flags.Strategy = flag.String("strategy", StrategyOnce, fmt.Sprintf("The Gonsul operation mode (%s, %s, %s, %s)", StrategyDry, StrategyOnce, StrategyPoll, StrategyHook)) + flags.Flags.RepoURL = flag.String("repo-url", "", "The repository URL (Full URL with scheme)") + flags.Flags.RepoSSHKey = flag.String("repo-ssh-key", "", "The SSH private key location (Full path)") + flags.Flags.RepoSSHUser = flag.String("repo-ssh-user", "git", "The SSH user name") + flags.Flags.RepoBranch = flag.String("repo-branch", "master", "Which branch should we look at") + flags.Flags.RepoRemoteName = flag.String("repo-remote-name", "origin", "The repository remote name") + flags.Flags.RepoBasePath = flag.String("repo-base-path", "/", "The base directory to look from inside the repo") + flags.Flags.RepoRootDir = flag.String("repo-root", "/tmp/gonsul/repo", "The path where the repo will be downloaded to") + flags.Flags.ConsulURL = flag.String("consul-url", "", "(REQUIRED) The Consul URL REST API endpoint (Full URL with scheme)") + flags.Flags.ConsulACL = flag.String("consul-acl", "", "The Consul ACL to use (Must have write on the KV following --consul-base path)") + flags.Flags.ConsulBasePath = flag.String("consul-base-path", "", "The base KV path will be prefixed to dir path - DO NOT START NOR END WITH SLASH") + flags.Flags.ExpandJSON = flag.Bool("expand-json", false, "Expand and parse JSON files as full paths? (Default false)") + flags.Flags.SecretsFile = flag.String("secrets-file", "", "A key value json file with placeholders->secrets mapping, in order to do on the fly replace") + flags.Flags.AllowDeletes = flag.Bool("allow-deletes", false, "Show Gonsul issue deletes? (If not, nothing will be done and a report on conflicting deletes will be shown) (Default false)") + flags.Flags.PollInterval = flag.Int("poll-interval", 60, "The number of seconds for the repository polling interval") + flags.Flags.ValidExtensions = flag.String("input-ext", "json,txt,ini", "A comma separated list of file extensions valid as input") + + // Parse our command line flags + flag.Parse() + + return flags.Flags +} + +func NewFlagsParser() *ConfigFlagsParser { + return &ConfigFlagsParser{ + Flags: interfaces.ConfigFlags{}, + } +} diff --git a/exporter/directory.go b/exporter/directory.go index 53ee70c..ab23d7d 100644 --- a/exporter/directory.go +++ b/exporter/directory.go @@ -21,7 +21,7 @@ func processDir(directory string, localData map[string]string) { } else { filePath := directory + "/" + file.Name() ext := filepath.Ext(filePath) - if ext != ".json" && ext != ".txt" && ext != ".ini" { + if !isExtensionValid(ext) { continue } content, err := ioutil.ReadFile(filePath) // just pass the file name @@ -33,6 +33,17 @@ func processDir(directory string, localData map[string]string) { } } +// isExtensionValid checks if given file extensions is valid for processing +func isExtensionValid(extension string) bool { + for _, validExtension := range config.GetValidExtensions() { + if strings.Trim(extension, ".") == strings.Trim(validExtension, ".") { + return true + } + } + + return false +} + func parseFile(filePath string, value string, localData map[string]string) { // Extract our file extension and cleanup file path ext := filepath.Ext(filePath) diff --git a/importer/consul.go b/importer/consul.go index fa1329c..267e2a5 100644 --- a/importer/consul.go +++ b/importer/consul.go @@ -31,7 +31,9 @@ func processConsulTransaction(transactions []structs.ConsulTxn, client *http.Cli } // Set ACL token - req.Header.Set("X-Consul-Token", config.GetConsulACL()) + if config.GetConsulACL() != "" { + req.Header.Set("X-Consul-Token", config.GetConsulACL()) + } // Send the request via a client // Do sends an HTTP request and diff --git a/importer/helper.go b/importer/helper.go index 7f7a68f..2fdc7e1 100644 --- a/importer/helper.go +++ b/importer/helper.go @@ -78,8 +78,10 @@ func createLiveData(client *http.Client) map[string]string { errorutil.ExitError(errors.New("NewRequestGET: "+err.Error()), errorutil.ErrorFailedConsulConnection, &logger) } - // Set ACL token - req.Header.Set("X-Consul-Token", config.GetConsulACL()) + // Set ACL token (if given) + if config.GetConsulACL() != "" { + req.Header.Set("X-Consul-Token", config.GetConsulACL()) + } // Send the request via a client, Do sends an HTTP request and returns an HTTP response resp, err := client.Do(req) diff --git a/interfaces/configurations.go b/interfaces/configurations.go new file mode 100644 index 0000000..2f9ec9b --- /dev/null +++ b/interfaces/configurations.go @@ -0,0 +1,25 @@ +package interfaces + +type ConfigFlags struct { + LogLevel *string + Strategy *string + RepoURL *string + RepoSSHKey *string + RepoSSHUser *string + RepoBranch *string + RepoRemoteName *string + RepoBasePath *string + RepoRootDir *string + ConsulURL *string + ConsulACL *string + ConsulBasePath *string + ExpandJSON *bool + SecretsFile *string + AllowDeletes *bool + PollInterval *int + ValidExtensions *string +} + +type IConfigFlags interface { + Parse() ConfigFlags +} \ No newline at end of file diff --git a/main.go b/main.go index 860705f..bf2d701 100644 --- a/main.go +++ b/main.go @@ -22,7 +22,7 @@ func main() { func bootstrap() { // Build our configuration - config, err := configuration.GetConfig() + config, err := configuration.GetConfig(configuration.NewFlagsParser()) if err != nil { var logger = errorutil.NewLogger(0) errorutil.ExitError(err, errorutil.ErrorBadParams, logger) @@ -35,5 +35,5 @@ func bootstrap() { app.Start(config, logger) // We're still here, all went well, good bye - logger.PrintInfo("Quitting... bye 😀") + logger.PrintInfo("Quitting... bye.") } diff --git a/tests/mocks/.gitkeep b/tests/mocks/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/test-repo/dev/app1/application.ini b/tests/test-repo/dev/app1/application.ini new file mode 100644 index 0000000..341c433 --- /dev/null +++ b/tests/test-repo/dev/app1/application.ini @@ -0,0 +1,6 @@ +[memory] +amount=512 +max=1024 + +[cpu] +cores=8 \ No newline at end of file diff --git a/tests/test-repo/dev/app1/config.json b/tests/test-repo/dev/app1/config.json new file mode 100644 index 0000000..88b3d71 --- /dev/null +++ b/tests/test-repo/dev/app1/config.json @@ -0,0 +1,4 @@ +{ + "key1": "value1", + "key2": "value2" +} \ No newline at end of file diff --git a/tests/test-repo/dev/app1/db-pass.txt b/tests/test-repo/dev/app1/db-pass.txt new file mode 100644 index 0000000..5f7b374 --- /dev/null +++ b/tests/test-repo/dev/app1/db-pass.txt @@ -0,0 +1 @@ +afunkypassword \ No newline at end of file diff --git a/tests/test-repo/dev/app1/gateways.yaml b/tests/test-repo/dev/app1/gateways.yaml new file mode 100644 index 0000000..565868e --- /dev/null +++ b/tests/test-repo/dev/app1/gateways.yaml @@ -0,0 +1,8 @@ +client1: + host: "http://asuperclient.com" + user: "thesuperuser" + pass: "thesuperpass" +client2: + host: "http://asuperclient.com" + user: "thesuperuser" + pass: "thesuperpass" \ No newline at end of file diff --git a/tests/test-repo/prod/app1/application.ini b/tests/test-repo/prod/app1/application.ini new file mode 100644 index 0000000..341c433 --- /dev/null +++ b/tests/test-repo/prod/app1/application.ini @@ -0,0 +1,6 @@ +[memory] +amount=512 +max=1024 + +[cpu] +cores=8 \ No newline at end of file diff --git a/tests/test-repo/prod/app1/config.json b/tests/test-repo/prod/app1/config.json new file mode 100644 index 0000000..88b3d71 --- /dev/null +++ b/tests/test-repo/prod/app1/config.json @@ -0,0 +1,4 @@ +{ + "key1": "value1", + "key2": "value2" +} \ No newline at end of file diff --git a/tests/test-repo/prod/app1/db-pass.txt b/tests/test-repo/prod/app1/db-pass.txt new file mode 100644 index 0000000..5f7b374 --- /dev/null +++ b/tests/test-repo/prod/app1/db-pass.txt @@ -0,0 +1 @@ +afunkypassword \ No newline at end of file diff --git a/tests/test-repo/prod/app1/gateways.yaml b/tests/test-repo/prod/app1/gateways.yaml new file mode 100644 index 0000000..565868e --- /dev/null +++ b/tests/test-repo/prod/app1/gateways.yaml @@ -0,0 +1,8 @@ +client1: + host: "http://asuperclient.com" + user: "thesuperuser" + pass: "thesuperpass" +client2: + host: "http://asuperclient.com" + user: "thesuperuser" + pass: "thesuperpass" \ No newline at end of file diff --git a/tests/test-repo/stg/app1/application.ini b/tests/test-repo/stg/app1/application.ini new file mode 100644 index 0000000..341c433 --- /dev/null +++ b/tests/test-repo/stg/app1/application.ini @@ -0,0 +1,6 @@ +[memory] +amount=512 +max=1024 + +[cpu] +cores=8 \ No newline at end of file diff --git a/tests/test-repo/stg/app1/config.json b/tests/test-repo/stg/app1/config.json new file mode 100644 index 0000000..88b3d71 --- /dev/null +++ b/tests/test-repo/stg/app1/config.json @@ -0,0 +1,4 @@ +{ + "key1": "value1", + "key2": "value2" +} \ No newline at end of file diff --git a/tests/test-repo/stg/app1/db-pass.txt b/tests/test-repo/stg/app1/db-pass.txt new file mode 100644 index 0000000..5f7b374 --- /dev/null +++ b/tests/test-repo/stg/app1/db-pass.txt @@ -0,0 +1 @@ +afunkypassword \ No newline at end of file diff --git a/tests/test-repo/stg/app1/gateways.yaml b/tests/test-repo/stg/app1/gateways.yaml new file mode 100644 index 0000000..565868e --- /dev/null +++ b/tests/test-repo/stg/app1/gateways.yaml @@ -0,0 +1,8 @@ +client1: + host: "http://asuperclient.com" + user: "thesuperuser" + pass: "thesuperpass" +client2: + host: "http://asuperclient.com" + user: "thesuperuser" + pass: "thesuperpass" \ No newline at end of file diff --git a/tests/test-secrets-file-fail.json b/tests/test-secrets-file-fail.json new file mode 100644 index 0000000..50b7ab5 --- /dev/null +++ b/tests/test-secrets-file-fail.json @@ -0,0 +1,3 @@ +{ + "WRONG_JSON_FORMAT": ["PLACEHOLDER","SECRET"] +} \ No newline at end of file diff --git a/tests/test-secrets-file-success.json b/tests/test-secrets-file-success.json new file mode 100644 index 0000000..6856339 --- /dev/null +++ b/tests/test-secrets-file-success.json @@ -0,0 +1,3 @@ +{ + "PLACEHOLDER": "SECRET" +} \ No newline at end of file From 5d3347a98dbdb8e87418f910a527193d80c21cc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Henri=20GU=C3=89RIN?= Date: Fri, 22 Feb 2019 11:35:30 +0100 Subject: [PATCH 2/5] :sparkles: Fail and log if error different than 404 --- importer/helper.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/importer/helper.go b/importer/helper.go index 2fdc7e1..2fe29a7 100644 --- a/importer/helper.go +++ b/importer/helper.go @@ -95,7 +95,9 @@ func createLiveData(client *http.Client) map[string]string { // Invalid response, path is empty then, fresh import if resp.StatusCode == 404 { return nil - } + } else if resp.StatusCode >= 400 { + errorutil.ExitError(errors.New("GetStatus: "+resp.Status), errorutil.ErrorFailedConsulConnection, &logger) + } // Read response from HTTP Response bodyBytes, err := ioutil.ReadAll(resp.Body) From 9c87f20732823a9dbb115dda1ddecad65e9248ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Henri=20GU=C3=89RIN?= Date: Fri, 22 Feb 2019 16:18:15 +0100 Subject: [PATCH 3/5] :art: Make it look better --- importer/helper.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/importer/helper.go b/importer/helper.go index 2fe29a7..d6b4780 100644 --- a/importer/helper.go +++ b/importer/helper.go @@ -95,8 +95,10 @@ func createLiveData(client *http.Client) map[string]string { // Invalid response, path is empty then, fresh import if resp.StatusCode == 404 { return nil - } else if resp.StatusCode >= 400 { - errorutil.ExitError(errors.New("GetStatus: "+resp.Status), errorutil.ErrorFailedConsulConnection, &logger) + } + + if resp.StatusCode >= 400 { + errorutil.ExitError(errors.New("Invalid response from consul: "+resp.Status), errorutil.ErrorFailedConsulConnection, &logger) } // Read response from HTTP Response From 48a0729269436a8f186319eec31ab1fca83ecb41 Mon Sep 17 00:00:00 2001 From: nritholtz Date: Tue, 25 Jun 2019 12:02:14 -0400 Subject: [PATCH 4/5] Add ability to provide http timeout for client via CLI option --- README.md | 9 +++++++++ configuration/config.go | 6 ++++++ configuration/config_test.go | 1 + configuration/flags.go | 1 + importer/importer.go | 2 +- interfaces/configurations.go | 1 + 6 files changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 60af51d..a8fa03a 100644 --- a/README.md +++ b/README.md @@ -308,6 +308,15 @@ This is the number of seconds you want Gonsul to wait between checks on the repo This is the file extensions that Gonsul should consider as inputs to populate our Consul. Please set each extension without the dot, and separate each extension with a comma. + +### `--timeout` +> `require:` **no** +> `default:` **5** +> `example:` **`--poll-interval=20`** + +The number of seconds for the client to wait for a response from Consul + + ## Gonsul Exit Codes Whenever an error occurs, and Gonsul exits with a code other than 0, we try to return a meaningful code, such as: diff --git a/configuration/config.go b/configuration/config.go index 7e38f7a..75de656 100644 --- a/configuration/config.go +++ b/configuration/config.go @@ -41,6 +41,7 @@ type Config struct { pollInterval int Working chan bool validExtensions []string + timeout int } func GetConfig(flagParser interfaces.IConfigFlags) (*Config, error) { @@ -131,6 +132,7 @@ func buildConfig(flags interfaces.ConfigFlags) (*Config, error) { pollInterval: *flags.PollInterval, Working: make(chan bool, 1), validExtensions: extensions, + timeout: *flags.Timeout, }, nil } @@ -210,6 +212,10 @@ func (config *Config) GetValidExtensions() []string { return config.validExtensions } +func (config *Config) GetTimeout() int { + return config.timeout +} + func buildSecretsMap(secretsFile string, repoRootPath string) (map[string]string, error) { var file = secretsFile if _, err := os.Stat(file); os.IsNotExist(err) { diff --git a/configuration/config_test.go b/configuration/config_test.go index 51c2e2e..bbf1d06 100644 --- a/configuration/config_test.go +++ b/configuration/config_test.go @@ -111,6 +111,7 @@ func getConfigFlagsFor( AllowDeletes: &ad, PollInterval: &pi, ValidExtensions: &ie, + Timeout: &ti, } return configFlags diff --git a/configuration/flags.go b/configuration/flags.go index 800ca50..eb8afb6 100644 --- a/configuration/flags.go +++ b/configuration/flags.go @@ -31,6 +31,7 @@ func (flags *ConfigFlagsParser) Parse() interfaces.ConfigFlags { flags.Flags.AllowDeletes = flag.Bool("allow-deletes", false, "Show Gonsul issue deletes? (If not, nothing will be done and a report on conflicting deletes will be shown) (Default false)") flags.Flags.PollInterval = flag.Int("poll-interval", 60, "The number of seconds for the repository polling interval") flags.Flags.ValidExtensions = flag.String("input-ext", "json,txt,ini", "A comma separated list of file extensions valid as input") + flags.Flags.Timeout = flag.Int("timeout", 5, "The number of seconds for the client to wait for a response from Consul") // Parse our command line flags flag.Parse() diff --git a/importer/importer.go b/importer/importer.go index cf3a952..f384cdc 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -29,7 +29,7 @@ func Start(localData map[string]string, conf *configuration.Config, log *errorut // create a Client // A Client is an HTTP client client := &http.Client{ - Timeout: time.Second * 5, + Timeout: time.Second * time.Duration(conf.GetTimeout()), } // Populate our Consul live data diff --git a/interfaces/configurations.go b/interfaces/configurations.go index 2f9ec9b..d904bf8 100644 --- a/interfaces/configurations.go +++ b/interfaces/configurations.go @@ -18,6 +18,7 @@ type ConfigFlags struct { AllowDeletes *bool PollInterval *int ValidExtensions *string + Timeout *int } type IConfigFlags interface { From 0f46c1596c767b91656bceabc7c46eb2a5f9a236 Mon Sep 17 00:00:00 2001 From: nritholtz Date: Tue, 25 Jun 2019 12:19:26 -0400 Subject: [PATCH 5/5] Fix README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a8fa03a..35a210e 100644 --- a/README.md +++ b/README.md @@ -312,7 +312,7 @@ without the dot, and separate each extension with a comma. ### `--timeout` > `require:` **no** > `default:` **5** -> `example:` **`--poll-interval=20`** +> `example:` **`--timeout=20`** The number of seconds for the client to wait for a response from Consul