From d0aa0b08a4ee964dfc4bae8a1d8211b291a5b447 Mon Sep 17 00:00:00 2001 From: francoispqt Date: Sat, 19 Jan 2019 02:07:19 +0800 Subject: [PATCH] initial commit --- .gitignore | 4 + .travis.yml | 16 + Gopkg.lock | 471 +++++++++++++++++++++++++++ Gopkg.toml | 38 +++ LICENSE | 19 ++ Makefile | 30 ++ README.md | 485 ++++++++++++++++++++++++++++ closers.go | 22 ++ config.go | 311 ++++++++++++++++++ config_test.go | 377 +++++++++++++++++++++ getter.go | 14 + getter_test.go | 23 ++ go.mod | 39 +++ go.sum | 88 +++++ group.go | 23 ++ group_test.go | 24 ++ loader.go | 226 +++++++++++++ loader/klenv/README.md | 34 ++ loader/klenv/envloader.go | 109 +++++++ loader/klenv/envloader_test.go | 132 ++++++++ loader/kletcd/README.md | 31 ++ loader/kletcd/etcdloader.go | 179 ++++++++++ loader/kletcd/etcdloader_test.go | 382 ++++++++++++++++++++++ loader/klfile/README.md | 25 ++ loader/klfile/fileloader.go | 178 ++++++++++ loader/klfile/fileloader_test.go | 208 ++++++++++++ loader/klflag/README.md | 16 + loader/klflag/flagloader.go | 77 +++++ loader/klflag/flagloader_test.go | 70 ++++ loader/klhttp/README.md | 19 ++ loader/klhttp/httploader.go | 154 +++++++++ loader/klhttp/httploader_test.go | 386 ++++++++++++++++++++++ loader/klhttp/source.go | 44 +++ loader/klvault/README.md | 21 ++ loader/klvault/auth/k8s/k8s.go | 171 ++++++++++ loader/klvault/auth/k8s/k8s_test.go | 235 ++++++++++++++ loader/klvault/auth/token/token.go | 14 + loader/klvault/authprovider.go | 8 + loader/klvault/vaultloader.go | 208 ++++++++++++ loader/klvault/vaultloader_test.go | 301 +++++++++++++++++ loader_mock_test.go | 90 ++++++ loader_test.go | 280 ++++++++++++++++ loaderwatcher.go | 40 +++ metrics.go | 79 +++++ metrics_test.go | 1 + mocks/authprovider_mock.go | 50 +++ mocks/client_mock.go | 49 +++ mocks/contexter_mock.go | 80 +++++ mocks/getter_mock.go | 86 +++++ mocks/kv_mock.go | 144 +++++++++ mocks/loader_mock.go | 91 ++++++ mocks/logicalclient_mock.go | 79 +++++ mocks/parser_mock.go | 49 +++ mocks/watcher_mock.go | 103 ++++++ parser/kpjson/README.md | 29 ++ parser/kpjson/jsonparser.go | 26 ++ parser/kpjson/jsonparser_test.go | 109 +++++++ parser/kpkeyval/kvparser.go | 63 ++++ parser/kpkeyval/kvparser_test.go | 90 ++++++ parser/kpmap/mapparser.go | 39 +++ parser/kpmap/mapparser_test.go | 1 + parser/kptoml/README.md | 24 ++ parser/kptoml/tomlparser.go | 24 ++ parser/kptoml/tomlparser_test.go | 113 +++++++ parser/kpyaml/README.md | 25 ++ parser/kpyaml/yamlparser.go | 25 ++ parser/kpyaml/yamlparser_test.go | 119 +++++++ parser/parser.go | 22 ++ parser/parser_test.go | 19 ++ test.sh | 12 + test/configfile_test.go | 89 +++++ test/data/cfg | 3 + test/data/cfg.yml | 15 + util.go | 264 +++++++++++++++ util_test.go | 269 +++++++++++++++ value.go | 288 +++++++++++++++++ value_test.go | 211 ++++++++++++ watcher.go | 65 ++++ watcher/kwfile/filewatcher.go | 122 +++++++ watcher/kwpoll/pollwatcher.go | 186 +++++++++++ watcher/kwpoll/pollwatcher_test.go | 66 ++++ watcher_mock_test.go | 103 ++++++ 82 files changed, 8854 insertions(+) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 Gopkg.lock create mode 100644 Gopkg.toml create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 closers.go create mode 100644 config.go create mode 100644 config_test.go create mode 100644 getter.go create mode 100644 getter_test.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 group.go create mode 100644 group_test.go create mode 100644 loader.go create mode 100644 loader/klenv/README.md create mode 100644 loader/klenv/envloader.go create mode 100644 loader/klenv/envloader_test.go create mode 100644 loader/kletcd/README.md create mode 100644 loader/kletcd/etcdloader.go create mode 100644 loader/kletcd/etcdloader_test.go create mode 100644 loader/klfile/README.md create mode 100644 loader/klfile/fileloader.go create mode 100644 loader/klfile/fileloader_test.go create mode 100644 loader/klflag/README.md create mode 100644 loader/klflag/flagloader.go create mode 100644 loader/klflag/flagloader_test.go create mode 100644 loader/klhttp/README.md create mode 100644 loader/klhttp/httploader.go create mode 100644 loader/klhttp/httploader_test.go create mode 100644 loader/klhttp/source.go create mode 100644 loader/klvault/README.md create mode 100644 loader/klvault/auth/k8s/k8s.go create mode 100644 loader/klvault/auth/k8s/k8s_test.go create mode 100644 loader/klvault/auth/token/token.go create mode 100644 loader/klvault/authprovider.go create mode 100644 loader/klvault/vaultloader.go create mode 100644 loader/klvault/vaultloader_test.go create mode 100644 loader_mock_test.go create mode 100644 loader_test.go create mode 100644 loaderwatcher.go create mode 100644 metrics.go create mode 100644 metrics_test.go create mode 100644 mocks/authprovider_mock.go create mode 100644 mocks/client_mock.go create mode 100644 mocks/contexter_mock.go create mode 100644 mocks/getter_mock.go create mode 100644 mocks/kv_mock.go create mode 100644 mocks/loader_mock.go create mode 100644 mocks/logicalclient_mock.go create mode 100644 mocks/parser_mock.go create mode 100644 mocks/watcher_mock.go create mode 100644 parser/kpjson/README.md create mode 100644 parser/kpjson/jsonparser.go create mode 100644 parser/kpjson/jsonparser_test.go create mode 100644 parser/kpkeyval/kvparser.go create mode 100644 parser/kpkeyval/kvparser_test.go create mode 100644 parser/kpmap/mapparser.go create mode 100644 parser/kpmap/mapparser_test.go create mode 100644 parser/kptoml/README.md create mode 100644 parser/kptoml/tomlparser.go create mode 100644 parser/kptoml/tomlparser_test.go create mode 100644 parser/kpyaml/README.md create mode 100644 parser/kpyaml/yamlparser.go create mode 100644 parser/kpyaml/yamlparser_test.go create mode 100644 parser/parser.go create mode 100644 parser/parser_test.go create mode 100755 test.sh create mode 100644 test/configfile_test.go create mode 100644 test/data/cfg create mode 100644 test/data/cfg.yml create mode 100644 util.go create mode 100644 util_test.go create mode 100644 value.go create mode 100644 value_test.go create mode 100644 watcher.go create mode 100644 watcher/kwfile/filewatcher.go create mode 100644 watcher/kwpoll/pollwatcher.go create mode 100644 watcher/kwpoll/pollwatcher_test.go create mode 100644 watcher_mock_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..45c2fa3e --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +debug.test +coverage.out +vendor/* +**/**/*.out diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..3095b463 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,16 @@ +language: go + +go: + - "1.10.x" + - "1.11.x" + - master + +script: + - go get -u golang.org/x/lint/golint + - make lint + - go get github.com/stretchr/testify github.com/golang/dep/cmd/dep + - dep ensure -v -vendor-only + - ./test.sh + +after_success: + - bash <(curl -s https://codecov.io/bash) diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 00000000..a7e7da01 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,471 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + digest = "1:e4b30804a381d7603b8a344009987c1ba351c26043501b23b8c7ce21f0b67474" + name = "github.com/BurntSushi/toml" + packages = ["."] + pruneopts = "" + revision = "3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005" + version = "v0.3.1" + +[[projects]] + branch = "master" + digest = "1:c0bec5f9b98d0bc872ff5e834fac186b807b656683bd29cb82fb207a1513fabb" + name = "github.com/beorn7/perks" + packages = ["quantile"] + pruneopts = "" + revision = "3a771d992973f24aa725d07868b467d1ddfceafb" + +[[projects]] + digest = "1:812ef5c0ff0dc70d100b57e2bec61aaf09b54873048f00bf685be3104943920b" + name = "github.com/coreos/etcd" + packages = [ + "auth/authpb", + "etcdserver/api/v3rpc/rpctypes", + "etcdserver/etcdserverpb", + "mvcc/mvccpb", + "pkg/types", + ] + pruneopts = "" + revision = "2cf9e51d2a78003b164c2998886158e60ded1cbb" + version = "v3.3.11" + +[[projects]] + digest = "1:0deddd908b6b4b768cfc272c16ee61e7088a60f7fe2f06c547bd3d8e1f8b8e77" + name = "github.com/davecgh/go-spew" + packages = ["spew"] + pruneopts = "" + revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" + version = "v1.1.1" + +[[projects]] + digest = "1:86c70c372aeec190cdccda2156f61e40c18c721f988b4a05fd2b1a1101b36580" + name = "github.com/francoispqt/gojay" + packages = ["."] + pruneopts = "" + revision = "f2cc13a668caf474b5d5806c7f1adbbe4ce28524" + version = "1.2.7" + +[[projects]] + digest = "1:fb6b4ea21671a4a8037cccc0a582caada51b4ac550219f42b9e8ce4671bc568b" + name = "github.com/go-test/deep" + packages = ["."] + pruneopts = "" + revision = "6592d9cc0a499ad2d5f574fde80a2b5c5cc3b4f5" + version = "v1.0.1" + +[[projects]] + digest = "1:527e1e468c5586ef2645d143e9f5fbd50b4fe5abc8b1e25d9f1c416d22d24895" + name = "github.com/gogo/protobuf" + packages = [ + "gogoproto", + "proto", + "protoc-gen-gogo/descriptor", + ] + pruneopts = "" + revision = "4cbf7e384e768b4e01799441fdf2a706a5635ae7" + version = "v1.2.0" + +[[projects]] + digest = "1:530233672f656641b365f8efb38ed9fba80e420baff2ce87633813ab3755ed6d" + name = "github.com/golang/mock" + packages = ["gomock"] + pruneopts = "" + revision = "51421b967af1f557f93a59e0057aaf15ca02e29c" + version = "v1.2.0" + +[[projects]] + digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18" + name = "github.com/golang/protobuf" + packages = [ + "proto", + "ptypes", + "ptypes/any", + "ptypes/duration", + "ptypes/timestamp", + ] + pruneopts = "" + revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" + version = "v1.2.0" + +[[projects]] + branch = "master" + digest = "1:2a5888946cdbc8aa360fd43301f9fc7869d663f60d5eedae7d4e6e5e4f06f2bf" + name = "github.com/golang/snappy" + packages = ["."] + pruneopts = "" + revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a" + +[[projects]] + digest = "1:8e3bd93036b4a925fe2250d3e4f38f21cadb8ef623561cd80c3c50c114b13201" + name = "github.com/hashicorp/errwrap" + packages = ["."] + pruneopts = "" + revision = "8a6fb523712970c966eefc6b39ed2c5e74880354" + version = "v1.0.0" + +[[projects]] + digest = "1:05334858a0cfb538622a066e065287f63f42bee26a7fda93a789674225057201" + name = "github.com/hashicorp/go-cleanhttp" + packages = ["."] + pruneopts = "" + revision = "e8ab9daed8d1ddd2d3c4efba338fe2eeae2e4f18" + version = "v0.5.0" + +[[projects]] + digest = "1:72308fdd6d5ef61106a95be7ca72349a5565809042b6426a3cfb61d99483b824" + name = "github.com/hashicorp/go-multierror" + packages = ["."] + pruneopts = "" + revision = "886a7fbe3eb1c874d46f623bfa70af45f425b3d1" + version = "v1.0.0" + +[[projects]] + digest = "1:84e2ab72e8daa944ec298efaea8c0be61abb50fefb332fa31aa5283029a59a7a" + name = "github.com/hashicorp/go-retryablehttp" + packages = ["."] + pruneopts = "" + revision = "d3a63d3c72068412f9ecd7c36faafd874d2e2888" + version = "v0.5.1" + +[[projects]] + branch = "master" + digest = "1:ff65bf6fc4d1116f94ac305342725c21b55c16819c2606adc8f527755716937f" + name = "github.com/hashicorp/go-rootcerts" + packages = ["."] + pruneopts = "" + revision = "6bb64b370b90e7ef1fa532be9e591a81c3493e00" + +[[projects]] + branch = "master" + digest = "1:d7b37f03787cb57693ded0aae8ef1420492fb94dab8e759bf389e04b99dd9212" + name = "github.com/hashicorp/go-sockaddr" + packages = ["."] + pruneopts = "" + revision = "e92cdb5343bbaf42b0a596937ae0f382270d6759" + +[[projects]] + digest = "1:d14365c51dd1d34d5c79833ec91413bfbb166be978724f15701e17080dc06dec" + name = "github.com/hashicorp/hcl" + packages = [ + ".", + "hcl/ast", + "hcl/parser", + "hcl/scanner", + "hcl/strconv", + "hcl/token", + "json/parser", + "json/scanner", + "json/token", + ] + pruneopts = "" + revision = "8cb6e5b959231cc1119e43259c4a608f9c51a241" + version = "v1.0.0" + +[[projects]] + digest = "1:067943f4fc285e3714bee874139fa8f6cf4a7a8cb7b615c96e6b0133c9af8d50" + name = "github.com/hashicorp/vault" + packages = [ + "api", + "helper/compressutil", + "helper/consts", + "helper/hclutil", + "helper/jsonutil", + "helper/parseutil", + "helper/strutil", + ] + pruneopts = "" + revision = "37a1dc9c477c1c68c022d2084550f25bf20cac33" + version = "v1.0.2" + +[[projects]] + branch = "master" + digest = "1:d0b151e4f07350afcca336335d5d3698e91ed95e9ffb54fe894806e8899590ef" + name = "github.com/jinzhu/copier" + packages = ["."] + pruneopts = "" + revision = "7e38e58719c33e0d44d585c4ab477a30f8cb82dd" + +[[projects]] + digest = "1:74fe97a8a2fa454ad9a70a4f1822ccbca39797a20432412d82d1742754fa3406" + name = "github.com/lalamove/nui" + packages = [ + "ncontext", + "nfs", + "ngetter", + "nlogger", + "nstrings", + ] + pruneopts = "" + revision = "7a3fba8943388cf350245bcdd5686d10e9d3bb1b" + version = "0.0.1" + +[[projects]] + digest = "1:63722a4b1e1717be7b98fc686e0b30d5e7f734b9e93d7dee86293b6deab7ea28" + name = "github.com/matttproud/golang_protobuf_extensions" + packages = ["pbutil"] + pruneopts = "" + revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" + version = "v1.0.1" + +[[projects]] + digest = "1:096a8a9182648da3d00ff243b88407838902b6703fc12657f76890e08d1899bf" + name = "github.com/mitchellh/go-homedir" + packages = ["."] + pruneopts = "" + revision = "ae18d6b8b3205b561c79e8e5f69bff09736185f4" + version = "v1.0.0" + +[[projects]] + digest = "1:bcc46a0fbd9e933087bef394871256b5c60269575bb661935874729c65bbbf60" + name = "github.com/mitchellh/mapstructure" + packages = ["."] + pruneopts = "" + revision = "3536a929edddb9a5b34bd6861dc4a9647cb459fe" + version = "v1.1.2" + +[[projects]] + digest = "1:b1df71d0b2287062b90c6b4c8d3c934440aa0d2eb201d03f22be0f045860b4aa" + name = "github.com/pierrec/lz4" + packages = [ + ".", + "internal/xxh32", + ] + pruneopts = "" + revision = "635575b42742856941dbc767b44905bb9ba083f6" + version = "v2.0.7" + +[[projects]] + digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411" + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + pruneopts = "" + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + digest = "1:6f218995d6a74636cfcab45ce03005371e682b4b9bee0e5eb0ccfd83ef85364f" + name = "github.com/prometheus/client_golang" + packages = [ + "prometheus", + "prometheus/internal", + ] + pruneopts = "" + revision = "505eaef017263e299324067d40ca2c48f6a2cf50" + version = "v0.9.2" + +[[projects]] + branch = "master" + digest = "1:adace4b303867385ba60abf9337de102689e96d47fc9a48c29fde202e12dd229" + name = "github.com/prometheus/client_model" + packages = ["go"] + pruneopts = "" + revision = "56726106282f1985ea77d5305743db7231b0c0a8" + +[[projects]] + branch = "master" + digest = "1:43465e4460e21f6462de4dbfdd864a2a48b743b25175d436932fb3e09765c937" + name = "github.com/prometheus/common" + packages = [ + "expfmt", + "internal/bitbucket.org/ww/goautoneg", + "model", + ] + pruneopts = "" + revision = "2998b132700a7d019ff618c06a234b47c1f3f681" + +[[projects]] + branch = "master" + digest = "1:b9f39c2f3278c4c3e9901d9b6a9b0316a1cb54fa6a0367e3498a8e8babc773b3" + name = "github.com/prometheus/procfs" + packages = [ + ".", + "internal/util", + "nfs", + "xfs", + ] + pruneopts = "" + revision = "bf6a532e95b1f7a62adf0ab5050a5bb2237ad2f4" + +[[projects]] + digest = "1:5b86241cd5275d19f32c57e8d116e298680e0cb8b0dd76cdb85e4272974f093d" + name = "github.com/radovskyb/watcher" + packages = ["."] + pruneopts = "" + revision = "3818ec23ec59ea15084fe26bfb114b3bb58aa132" + version = "v1.0.5" + +[[projects]] + digest = "1:29df111893b87bd947307aab294c042e900c2f29c53ad3896127955b4283728a" + name = "github.com/ryanuber/go-glob" + packages = ["."] + pruneopts = "" + revision = "572520ed46dbddaed19ea3d9541bdd0494163693" + version = "v0.1" + +[[projects]] + digest = "1:ae3493c780092be9d576a1f746ab967293ec165e8473425631f06658b6212afc" + name = "github.com/spf13/cast" + packages = ["."] + pruneopts = "" + revision = "8c9545af88b134710ab1cd196795e7f2388358d7" + version = "v1.3.0" + +[[projects]] + digest = "1:381bcbeb112a51493d9d998bbba207a529c73dbb49b3fd789e48c63fac1f192c" + name = "github.com/stretchr/testify" + packages = [ + "assert", + "require", + ] + pruneopts = "" + revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053" + version = "v1.3.0" + +[[projects]] + digest = "1:812ef5c0ff0dc70d100b57e2bec61aaf09b54873048f00bf685be3104943920b" + name = "go.etcd.io/etcd" + packages = ["clientv3"] + pruneopts = "" + revision = "2cf9e51d2a78003b164c2998886158e60ded1cbb" + version = "v3.3.11" + +[[projects]] + branch = "master" + digest = "1:7ec13687f85b25087fe05f6ea8dd116013a8263f8eb7e057da7664bc7599d2d4" + name = "golang.org/x/net" + packages = [ + "context", + "http/httpguts", + "http2", + "http2/hpack", + "idna", + "internal/timeseries", + "trace", + ] + pruneopts = "" + revision = "915654e7eabcea33ae277abbecf52f0d8b7a9fdc" + +[[projects]] + branch = "master" + digest = "1:057806b6e218f4ceac80d27ccd8dd80b2c06b0050289aa46e0edfd82cc4faa2c" + name = "golang.org/x/sys" + packages = ["unix"] + pruneopts = "" + revision = "11f53e03133963fb11ae0588e08b5e0b85be8be5" + +[[projects]] + digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4" + name = "golang.org/x/text" + packages = [ + "collate", + "collate/build", + "internal/colltab", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "language", + "secure/bidirule", + "transform", + "unicode/bidi", + "unicode/cldr", + "unicode/norm", + "unicode/rangetable", + ] + pruneopts = "" + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" + +[[projects]] + branch = "master" + digest = "1:14cb1d4240bcbbf1386ae763957e04e2765ec4e4ce7bb2769d05fa6faccd774e" + name = "golang.org/x/time" + packages = ["rate"] + pruneopts = "" + revision = "85acf8d2951cb2a3bde7632f9ff273ef0379bcbd" + +[[projects]] + branch = "master" + digest = "1:1aa609a0033ef2927e083f2e5e07203ca35fe21c4a24b563de9fea16ddaae9ba" + name = "google.golang.org/genproto" + packages = ["googleapis/rpc/status"] + pruneopts = "" + revision = "db91494dd46c1fdcbbde05e5ff5eb56df8f7d79a" + +[[projects]] + digest = "1:39d4d828b87d58d114fdc211f0638f32dcae84019fe17d6b48f9b697f4b60213" + name = "google.golang.org/grpc" + packages = [ + ".", + "balancer", + "balancer/base", + "balancer/roundrobin", + "binarylog/grpc_binarylog_v1", + "codes", + "connectivity", + "credentials", + "credentials/internal", + "encoding", + "encoding/proto", + "grpclog", + "health/grpc_health_v1", + "internal", + "internal/backoff", + "internal/binarylog", + "internal/channelz", + "internal/envconfig", + "internal/grpcrand", + "internal/grpcsync", + "internal/syscall", + "internal/transport", + "keepalive", + "metadata", + "naming", + "peer", + "resolver", + "resolver/dns", + "resolver/passthrough", + "stats", + "status", + "tap", + ] + pruneopts = "" + revision = "a02b0774206b209466313a0b525d2c738fe407eb" + version = "v1.18.0" + +[[projects]] + digest = "1:cedccf16b71e86db87a24f8d4c70b0a855872eb967cb906a66b95de56aefbd0d" + name = "gopkg.in/yaml.v2" + packages = ["."] + pruneopts = "" + revision = "51d6538a90f86fe93ac480b35f37b2be17fef232" + version = "v2.2.2" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + input-imports = [ + "github.com/BurntSushi/toml", + "github.com/coreos/etcd/mvcc/mvccpb", + "github.com/francoispqt/gojay", + "github.com/go-test/deep", + "github.com/golang/mock/gomock", + "github.com/hashicorp/go-multierror", + "github.com/hashicorp/vault/api", + "github.com/jinzhu/copier", + "github.com/lalamove/nui/ncontext", + "github.com/lalamove/nui/nfs", + "github.com/lalamove/nui/ngetter", + "github.com/lalamove/nui/nlogger", + "github.com/lalamove/nui/nstrings", + "github.com/prometheus/client_golang/prometheus", + "github.com/radovskyb/watcher", + "github.com/spf13/cast", + "github.com/stretchr/testify/require", + "go.etcd.io/etcd/clientv3", + "gopkg.in/yaml.v2", + ] + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 00000000..ec616141 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,38 @@ +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + + +[[constraint]] + name = "go.etcd.io/etcd" + version = "3.3.10" + +[[constraint]] + name = "github.com/spf13/cast" + version = "1.3.0" + +[[constraint]] + name = "github.com/lalamove/nui" + version = "0.0.1" diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..ea696ba6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2019 Lalamove + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..4b5a82ef --- /dev/null +++ b/Makefile @@ -0,0 +1,30 @@ +test: + go test $(shell go list ./... | grep -v /examples/ ) -covermode=count + +test-race: + go test -race $(shell go list ./... | grep -v /examples/ ) + +coverage: + GO111MODULE=off go test ./... -cover -covermode=count -coverprofile=cover.out; GO111MODULE=off go tool cover -func cover.out; + +coverage-html: + GO111MODULE=off go test ./... -cover -covermode=count -coverprofile=cover.out; GO111MODULE=off go tool cover -html=cover.out; + +lint: + golint -set_exit_status $(shell (go list ./... | grep -v /vendor/)) + +mocks: + mockgen -source ./loader.go -package mocks > ./mocks/loader_mock.go + mockgen -source ./watcher.go -package mocks > ./mocks/watcher_mock.go + mockgen -source ./loader.go -package konfig > ./loader_mock_test.go + mockgen -source ./watcher.go -package konfig > ./watcher_mock_test.go + mockgen -source ./loader/klvault/authprovider.go -package mocks > ./mocks/authprovider_mock.go + mockgen -source ./loader/klvault/vaultloader.go -package mocks LogicalClient > ./mocks/logicalclient_mock.go + mockgen -source ./parser/parser.go -package mocks Parser > ./mocks/parser_mock.go + mockgen -source ./loader/klhttp/httploader.go -package mocks Client > ./mocks/client_mock.go + mockgen -source ./watcher/kwpoll/pollwatcher.go -package mocks Getter > ./mocks/getter_mock.go + mockgen -package mocks go.etcd.io/etcd/clientv3 KV > ./mocks/kv_mock.go + mockgen -package mocks github.com/lalamove/nui/ncontext Contexter > ./mocks/contexter_mock.go + mockgen -source ./parser/parser.go -package mocks Parser > ./mocks/parser_mock.go + +.PHONY: test test-race coverage coverage-html lint bench mocks diff --git a/README.md b/README.md new file mode 100644 index 00000000..177c07d8 --- /dev/null +++ b/README.md @@ -0,0 +1,485 @@ +[![Build Status](https://travis-ci.org/lalamove/konfig.svg?branch=master)](https://travis-ci.org/lalamove/konfig) +[![codecov](https://codecov.io/gh/lalamove/konfig/branch/master/graph/badge.svg)](https://codecov.io/gh/lalamove/konfig) +[![Go Report Card](https://goreportcard.com/badge/github.com/lalamove/konfig)](https://goreportcard.com/report/github.com/lalamove/konfig) +[![Go doc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square +)](https://godoc.org/github.com/lalamove/konfig) + +# Konfig +Composable, dynamic, observable configurations for Go. +It allows you to compose configurations from multiple sources with reload mechanism making it simple to build stateful apps. + +# Why another config package? +Most config package for Golang are hardly extensible and composable and make it complex to build apps which can reload their state dynamicaly. Also, few of them come with sources such as Vault, Etcd and multiple encoding formats. + +konfig is built around 4 small interfaces: +- Loader +- Watcher +- Parser +- Closer + +Konfig features include: +- **Dynamic** configuration loading +- **Composable** load configs from multiple sources. At Lalamove we load configs from Vault, files and etcd. +- **Polyglot** load configs from multiple format. Konfig support JSON, YAML, TOML, Key=Value. +- **Fast, Lockfree, Thread safe Read** +- **Observable config, Hot Reload** mechanism and tooling to manage state. +- **Typed Read** get typed values from config or bind a struct. +- **Metrics** exposed prometheus metrics telling you how many times a config is reloaded, if it failed, and how long it takes to reload! + +# Get started +```sh +go get github.com/lalamove/konfig +``` + +Load and watch a json formatted config file. +```go +var configFiles = []klfile.File{ + { + Path: "./konfig.json", + Parser: json.Parser, + }, +} + +func init() { + konfig.Init(konfig.DefaultConfig()) +} + +func main() { + // load from json file + konfig.RegisterLoaderWatcher( + klfile.New(&klfile.Config{ + Files: configFiles, + Watch: true, + }), + // optionally you can pass config hooks to run when a file is changed + func(c Store) error { + return nil + }, + ) + + if err := konfig.Watch(); err != nil { + log.Fatal(err) + } + + // retrieve value from config file + konfig.Bool("debug") +} +``` + +# Store +The Store is the base of the config package. It holds and gives access to values stored by keys. + +## Creating a Store +You can create a global Store by calling `konfig.Init(*konfig.Config)`: +```go +konfig.Init(konfig.DefaultConfig()) +``` +The global store is accessible directly from the package: +```go +konfig.Get("foo") // calls store.Get("foo") +``` + +You can create a new store by calling `konfig.New(*konfig.Config)`: +```go +s := konfig.New(konfig.DefaultConfig()) +``` + +## Loading and Watching a Store +After registering Loaders and Watchers in the konfig.Store, you must load and watch the store. + +You can do both by calling `LoadWatch`: +```go +if err := konfig.LoadWatch(); err != nil { + log.Fatal(err) +} +``` + +You can call `Load` onlyi, it will load all loaders and return: +```go +if err := konfig.Load(); err != nil { + log.Fatal(err) +} +``` + +And finally you can call `Watch` only, it will start all watchers and return: +```go +if err := konfig.Watch(); err != nil { + log.Fatal(err) +} +``` + + +# Loaders +Loaders load config values into the store. A loader is an implementation of the loader interface. +```go +type Loader interface { + // Loads the config and add it to the Store + Load(Store) error + // MaxRetry returns the maximum number of times to allow retrying on load failure + MaxRetry() int + // RetryDelay returns the delay to wait before retrying + RetryDelay() time.Duration +} +``` +You can register loaders in the config individually or with a watcher. + +### Register a loader by itself: +```go +configLoader := konfig.RegisterLoader( + klfile.New( + &klfile.Config{ + Files: []klfile.File{ + { + Parser: kpjson.Parser, + Path: "./konfig.json", + }, + }, + }, + ), +) +``` + +### Register a loader with a watcher: +To register a loader and a watcher together, you must register a LoaderWatcher which is an interface that implements both the Loader and the Watcher interface. +```go +configLoader := konfig.RegisterLoaderWatcher( + klfile.New( + &klfile.Config{ + Files: []klfile.File{ + { + Parser: kpjson.Parser, + Path: "./konfig.json", + }, + }, + Watch: true, + }, + ), +) +``` +You can also compose a loader and a watcher to create a LoaderWatcher: +```go +configLoader := konfig.RegisterLoaderWatcher( + // it creates a LoaderWatcher from a loader and a watcher + konfig.NewLoaderWatcher( + someLoader, + someWatcher, + ), +) +``` +### Built in loaders +Config already has the following loaders, they all have a built in watcher: +- [File Loader](loader/klfile/README.md) + +Loads configs from files which can be watched. Files can have different parsers to load different formats. It has a built in file watcher which triggers a config reload (running hooks) when files are modified. + +- [Vault Loader](loader/klvault/README.md) + +Loads configs from vault secrets. It has a built in Poll Watcher which triggers a config reload (running hooks) before the secret and the token from the auth provider expires. + +- [HTTP Loader](loader/klhttp/README.md) + +Loads configs from HTTP sources. Sources can have different parsers to load different formats. It has a built in Poll Diff Watcher which triggers a config reload (running hooks) if data is different. + +- [Etcd Loader](loader/kletcd/README.md) + +Loads configs from Etcd keys. Keys can have different parser to load different formats. It has a build in Poll Diff Watcher which triggers a config reload (running hooks) if data is different. + +- [ENV Loader](loader/klenv/README.md) + +Loads configs from environment variables. + +- [Flag Loader](loader/klflag/README.md) + +Loads configs from command line flags. + + +### Parsers +Parsers parse an io.Reader into a konfig.Store. There are used by some loaders to parse the data they fetch into the config store. the File Loader, Etcd Loader and HTTP Loader use Parsers. + +Config already has the following parsers: +- [JSON Parser](parser/kpjson/README.md) +- [TOML Parser](parser/kptoml/README.md) +- [YAML Parser](parser/kpyaml/README.md) +- [KV Parser](parser/kpkeyval/README.md) +- [Map Parser](parser/kpmap/README.md) + +# Watchers +Watchers trigger a call on a Loader on events. A watcher is an implementation of the Watcher interface. +```go +type Watcher interface { + // Start starts the watcher, it must not be blocking. + Start() error + // Done indicate wether the watcher is done or not + Done() <-chan struct{} + // Watch should block until an event unlocks it + Watch() <-chan struct{} + // Close closes the watcher, it returns a non nil error if it is already closed + // or something prevents it from closing properly. + Close() error +} +``` +### Built in watchers +Config already has the following watchers: +- [File Watcher](watcher/filewatcher/README.md) + +Watches files for changes. + +- [Poll Watcher](watcher/filewatcher/README.md) + +Sends events at a given rate, or if diff is enabled, it takes a Getter and fetches the data +at a given rate, if data is different, it sends an event. + +# Hooks +Hooks are functions ran after a successful loader Load call. They are used to reload the state of the application on a config change. + +### Registering a loader with some hooks +You can register a loader or a loader watcher with hooks. +```go +configLoader := konfig.RegisterLoaderWatcher( + klfile.New( + &klfile.Config{ + Files: []klfile.File{ + { + Parser: yamlparser.Parser, + Path: "./konfig.yaml", + }, + }, + Watch: true, + }, + ), + func(s konfig.Store) error { + // Here you should reload the state of your app + return nil + }, +) +``` + +### Adding hooks to an existing loader +You can register a loader or a loader watcher with hooks. +```go +configLoader.AddHooks( + func(s konfig.Store) error { + // Here you should reload the state of your app + return nil + }, + func(s konfig.Store) error { + // Here you should reload the state of your app + return nil + }, +) +``` + +# Closers +Closers can be added to the config so that if config fails to load it will close the registered Closers. +```go +type Closer interface { + Close() error +} +``` +## Register a Closer +```go +konfig.RegisterCloser(closer) +``` + +# Config groups +You can namespace your configs using config Groups. +```go +konfig.Group("db").RegisterLoaderWatcher( + klfile.New( + &klfile.Config{ + Files: []klfile.File{ + { + Parser: yamlparser.Parser, + Path: "./db.yaml", + }, + }, + Watch: true, + }, + ), +) + +// accessing grouped config +dbHost := konfig.Group("db").MustString("credentials.host") +``` + +# Binding a type to a Store +You can bind a type to the konfig store if you want your config values to be unmarshaled to a **struct** or a **map[string]interface{}**. Then you can access an instance of that type in a thread safe manner(in order to be safe for dynamic config updates). + +Let's see with an example of a json config file: +```json +{ + "addr": ":8080", + "debug": true, + "db": { + "username": "foo" + }, + "redis": { + "host": "127.0.0.1" + } +} +``` + +```go +type DBConfig struct { + Username string +} +type Config struct { + Addr string + Debug string + DB DBConfig `konfig:"db"` + RedisHost string `konfig:"redis.host"` +} + +// we init the root konfig store +konfig.Init(konfig.DefaultConfig()) + +// we bind the Config struct to the konfig.Store +konfig.Bind(Config{}) + +// we register our config file +konfig.RegisterLoaderWatcher( + klfile.New( + &klfile.Config{ + Files: []klfile.File{ + { + Parser: kpjson.Parser, + Path: "./config.json", + }, + }, + Watch: true, + }, + ), +) + +// we load our config and start watching +if err := konfig.LoadWatch(); err != nil { + log.Fatal(err) +} + +// Get our config value +c := konfig.Value().(Config) + +fmt.Println(c.Addr) // :8080 +``` + +Note that you can compose your config sources. For example, have your credentials come from Vault and be renewed often and have the rest of your config loaded from a file and be updated on file change. + +**It is important to understand how Konfig unmarshals your config values into your struct.** +When a Loader calls *konfig.Set()*, if the konfig store has a value bound to it, it will try to unmarshal the key to the bound value. +- First, it will look for field tags in the struct, if a tag matches exactly the key, it will unmarshal the key to the struct field. +- Then, it will do a EqualFold on the field name and the key, if they match, it will unmarshal the key to the struct field. +- Then, if the key has a dot, it will check if the tag or the field name (to lowercase) is a prefix of the key, if yes, it will check if the type of the field is a struct of pointer, if yes, it will check the struct using whats after the prefix as the key. + + +# Read from config +Apart from reading from the bound config value, konfig provides several methods to read values. + +Every method to retrieve config values come in 2 flavours: +- **Get** | reads a the value at the given key. If key is not present it returns the zero value of the type. +- **MustGet** | reads a the value at the given key. If key is not present it panics. + +All methods to read values from a Store: +```go +// Exists checks wether the key k is set in the store. +Exists(k string) bool + +// Get gets the value with the key k fron the store. If the key is not set, Get returns nil. To check wether a value is really set, use Exists. +Get(k string) interface{} +// MustGet tries to get the value with the key k from the store. If the key k does not exist in the store, MustGet panics. +MustGet(k string) interface{} + +// MustString tries to get the value with the key k from the store and casts it to a string. If the key k does not exist in the store, MustGet panics. +MustString(k string) string +// String tries to get the value with the key k from the store and casts it to a string. If the key k does not exist it returns the Zero value. +String(k string) string + +// MustInt tries to get the value with the key k from the store and casts it to a int. If the key k does not exist in the store, MustInt panics. +MustInt(k string) int +// Int tries to get the value with the key k from the store and casts it to a int. If the key k does not exist it returns the Zero value. +Int(k string) int + +// MustFloat tries to get the value with the key k from the store and casts it to a float. If the key k does not exist in the store, MustFloat panics. +MustFloat(k string) float64 +// Float tries to get the value with the key k from the store and casts it to a float. If the key k does not exist it returns the Zero value. +Float(k string) float64 + +// MustBool tries to get the value with the key k from the store and casts it to a bool. If the key k does not exist in the store, MustBool panics. +MustBool(k string) bool +// Bool tries to get the value with the key k from the store and casts it to a bool. If the key k does not exist it returns the Zero value. +Bool(k string) bool + +// MustDuration tries to get the value with the key k from the store and casts it to a time.Duration. If the key k does not exist in the store, MustDuration panics. +MustDuration(k string) time.Duration +// Duration tries to get the value with the key k from the store and casts it to a time.Duration. If the key k does not exist it returns the Zero value. +Duration(k string) time.Duration + +// MustTime tries to get the value with the key k from the store and casts it to a time.Time. If the key k does not exist in the store, MustTime panics. +MustTime(k string) time.Time +// Time tries to get the value with the key k from the store and casts it to a time.Time. If the key k does not exist it returns the Zero value. +Time(k string) time.Time + +// MustStringSlice tries to get the value with the key k from the store and casts it to a []string. If the key k does not exist in the store, MustStringSlice panics. +MustStringSlice(k string) []string +// StringSlice tries to get the value with the key k from the store and casts it to a []string. If the key k does not exist it returns the Zero value. +StringSlice(k string) []string + +// MustIntSlice tries to get the value with the key k from the store and casts it to a []int. If the key k does not exist in the store, MustIntSlice panics. +MustIntSlice(k string) []int +// IntSlice tries to get the value with the key k from the store and casts it to a []int. If the key k does not exist it returns the Zero value. +IntSlice(k string) []int + +// MustStringMap tries to get the value with the key k from the store and casts it to a map[string]interface{}. If the key k does not exist in the store, MustStringMap panics. +MustStringMap(k string) map[string]interface{} +// StringMap tries to get the value with the key k from the store and casts it to a map[string]interface{}. If the key k does not exist it returns the Zero value. +StringMap(k string) map[string]interface{} + +// MustStringMapString tries to get the value with the key k from the store and casts it to a map[string]string. If the key k does not exist in the store, MustStringMapString panics. +MustStringMapString(k string) map[string]string +// StringMapString tries to get the value with the key k from the store and casts it to a map[string]string. If the key k does not exist it returns the Zero value. +StringMapString(k string) map[string]string +``` + +# Getter +To easily build services which can use dynamically loaded configs you can create getters for specific keys. A getter implements `ngetter.GetterTyped` from [nui](github.com/lalamove/nui) package. It is useful when building stateful apps. + +Example with a config value set for the debug key: +```go +debug := konfig.Getter("debug") + +debug.Bool() // true +``` + +# Metrics +Konfig comes with prometheus metrics. + +Two metrics are exposed: +- Config reloads counter vector with labels +- Config reload duration summary vector with labels + +Example of metrics: +``` +# HELP konfig_loader_reload Number of config loader reload +# TYPE konfig_loader_reload counter +konfig_loader_reload{loader_name="config-files",result="failure",store_name=""} 0.0 +konfig_loader_reload{loader_name="config-files",result="success",store_name=""} 1.0 + +# HELP konfig_loader_reload_duration Histogram for the config reload duration +# TYPE konfig_loader_reload_duration summary +konfig_loader_reload_duration{loader_name="config-files",store_name="",quantile="0.5"} 0.001227641 +konfig_loader_reload_duration{loader_name="config-files",store_name="",quantile="0.9"} 0.001227641 +konfig_loader_reload_duration{loader_name="config-files",store_name="",quantile="0.99"} 0.001227641 +konfig_loader_reload_duration_sum{loader_name="config-files",store_name=""} 0.001227641 +konfig_loader_reload_duration_count{loader_name="config-files",store_name=""} 1.0 +``` + +To enable metrics, you must pass a custom config when creating a config store: +```go +konfig.Init(&konfig.Config{ + Metrics: true, + Name: "root", +}) +``` + +# Contributing + +Contributions are welcome. To make contributions, fork the repository, create a branch and submit a Pull Request to the master branch. diff --git a/closers.go b/closers.go new file mode 100644 index 00000000..1bd5c3ba --- /dev/null +++ b/closers.go @@ -0,0 +1,22 @@ +package konfig + +import ( + "io" + + multierror "github.com/hashicorp/go-multierror" +) + +// Closers is a multi closer +type Closers []io.Closer + +// Close closes all closers in the multi closer and returns an error if an error was encountered. +// Error returned is multierror.Error. https://github.com/hashicorp/go-multierror +func (cs Closers) Close() error { + var multiErr error + for _, closer := range cs { + if err := closer.Close(); err != nil { + multierror.Append(multiErr, err) + } + } + return multiErr +} diff --git a/config.go b/config.go new file mode 100644 index 00000000..c99bf68a --- /dev/null +++ b/config.go @@ -0,0 +1,311 @@ +package konfig + +import ( + "errors" + "io" + "os" + "sync" + "sync/atomic" + "time" + + "github.com/lalamove/nui/nlogger" + "github.com/prometheus/client_golang/prometheus" +) + +var _ Store = (*store)(nil) + +var ( + // ErrInvalidConfigFileFormat is the error returned when a problem is encountered when parsing the + // config file + ErrInvalidConfigFileFormat = errors.New("Err invalid file format") + + // ErrLoaderNotFound is the error thrown when the loader with the given name cannot be found in the config store + ErrLoaderNotFound = errors.New("Err loader not found") + // ErrConfigNotFoundMsg is the error message thrown when a config key is not set + ErrConfigNotFoundMsg = "Err config %s not found" +) + +const ( + missingConfMsg = "Config %s missing" + defaultName = "root" +) + +// ErrMissingConfig is the type representing an error when a required config is missing +type ErrMissingConfig string + +// Error implements the error interface +func (e ErrMissingConfig) Error() string { + return string(e) +} + +// DefaultConfig returns a default Config +func DefaultConfig() *Config { + return &Config{ + ExitCode: 1, + Logger: nlogger.New(os.Stdout, "CONFIG | "), + Name: defaultName, + } +} + +// Config is the config to init a config store +type Config struct { + // Name is the name of the config store, it is used in metrics as a label. When creating a config group, the name of the group becomes the name of the store + Name string + // ExitCode is the code to exit when errors are encountered in loaders + ExitCode int + // Disables exiting the program (os.Exit) when errors on loaders + NoExitOnError bool + // NoStopOnFailure if false the store closes all registered Watchers and Closers and exit the process unless NoExitOnError is true + // when a Loader fails to load or a Loader Hook fails. If true, nothing happens when a Loader fails. + NoStopOnFailure bool + // Logger is the logger used internally + Logger nlogger.Logger + // Metrics sets whether a konfig.Store should record metrics for config loaders + Metrics bool +} + +// Store is the interface +type Store interface { + // Name returns the name of the store + Name() string + // RegisterLoader registers a Loader in the store and adds the given loader hooks. + RegisterLoader(l Loader, loaderHooks ...func(Store) error) *ConfigLoader + // RegisterLoaderWatcher reigsters a LoaderWatcher in the store and adds the given loader hooks. + RegisterLoaderWatcher(lw LoaderWatcher, loaderHooks ...func(Store) error) *ConfigLoader + // RegisterCloser registers an io.Closer in the store. A closer closes when konfig fails to load configs. + RegisterCloser(closer io.Closer) Store + + // Load loads all loaders registered in the store. If it faisl it returns a non nil error + Load() error + // Watch starts all watchers registered in the store. If it fails it returns a non nil error. + Watch() error + + // LoadWatch loads all loaders registered in the store, then starts watching all + // watchers. If loading or starting watchers fails, loadwatch stops and returns a non nil error. + LoadWatch() error + + // Group lazyloads a child Store from the current store. If the group already exists, it just returns it, else it creates it and returns it. Groups are useful to namespace configs by domain. + Group(g string) Store + + // Get gets the value with the key k fron the store. If the key is not set, Get returns nil. To check wether a value is really set, use Exists. + Get(k string) interface{} + // MustGet tries to get the value with the key k from the store. If the key k does not exist in the store, MustGet panics. + MustGet(k string) interface{} + // Set sets the key k with the value v in the store. + Set(k string, v interface{}) + // Exists checks wether the key k is set in the store. + Exists(k string) bool + // MustString tries to get the value with the key k from the store and casts it to a string. If the key k does not exist in the store, MustGet panics. + MustString(k string) string + + // String tries to get the value with the key k from the store and casts it to a string. If the key k does not exist it returns the Zero value. + String(k string) string + + // MustInt tries to get the value with the key k from the store and casts it to a int. If the key k does not exist in the store, MustInt panics. + MustInt(k string) int + + // Int tries to get the value with the key k from the store and casts it to a int. If the key k does not exist it returns the Zero value. + Int(k string) int + + // MustFloat tries to get the value with the key k from the store and casts it to a float. If the key k does not exist in the store, MustFloat panics. + MustFloat(k string) float64 + // Float tries to get the value with the key k from the store and casts it to a float. If the key k does not exist it returns the Zero value. + Float(k string) float64 + + // MustBool tries to get the value with the key k from the store and casts it to a bool. If the key k does not exist in the store, MustBool panics. + MustBool(k string) bool + // Bool tries to get the value with the key k from the store and casts it to a bool. If the key k does not exist it returns the Zero value. + Bool(k string) bool + + // MustDuration tries to get the value with the key k from the store and casts it to a time.Duration. If the key k does not exist in the store, MustDuration panics. + MustDuration(k string) time.Duration + // Duration tries to get the value with the key k from the store and casts it to a time.Duration. If the key k does not exist it returns the Zero value. + Duration(k string) time.Duration + + // MustTime tries to get the value with the key k from the store and casts it to a time.Time. If the key k does not exist in the store, MustTime panics. + MustTime(k string) time.Time + // Time tries to get the value with the key k from the store and casts it to a time.Time. If the key k does not exist it returns the Zero value. + Time(k string) time.Time + + // MustStringSlice tries to get the value with the key k from the store and casts it to a []string. If the key k does not exist in the store, MustStringSlice panics. + MustStringSlice(k string) []string + // StringSlice tries to get the value with the key k from the store and casts it to a []string. If the key k does not exist it returns the Zero value. + StringSlice(k string) []string + + // MustIntSlice tries to get the value with the key k from the store and casts it to a []int. If the key k does not exist in the store, MustIntSlice panics. + MustIntSlice(k string) []int + // IntSlice tries to get the value with the key k from the store and casts it to a []int. If the key k does not exist it returns the Zero value. + IntSlice(k string) []int + + // MustStringMap tries to get the value with the key k from the store and casts it to a map[string]interface{}. If the key k does not exist in the store, MustStringMap panics. + MustStringMap(k string) map[string]interface{} + // StringMap tries to get the value with the key k from the store and casts it to a map[string]interface{}. If the key k does not exist it returns the Zero value. + StringMap(k string) map[string]interface{} + + // MustStringMapString tries to get the value with the key k from the store and casts it to a map[string]string. If the key k does not exist in the store, MustStringMapString panics. + MustStringMapString(k string) map[string]string + // StringMapString tries to get the value with the key k from the store and casts it to a map[string]string. If the key k does not exist it returns the Zero value. + StringMapString(k string) map[string]string + + // Bind binds a value (either a map[string]interface{} or a struct) to the config store. When config values are set on the config store, they are also set on the bound value. + Bind(interface{}) + + // Value returns the value bound to the config store. + // It panics if no bound value has been set + Value() interface{} +} + +// store is the concrete implementation of the Store +type store struct { + name string + cfg *Config + m *atomic.Value + mut *sync.Mutex + groups map[string]*store + v *value + metrics map[string]prometheus.Collector + + WatcherLoaders []*loaderWatcher + WatcherClosers Closers + Closers Closers +} + +var ( + c *store + once sync.Once +) + +// Init initiates the global config store with the given Config cfg +func Init(cfg *Config) { + c = newStore(cfg) +} + +// New returns a new Store with the given config +func New(cfg *Config) Store { + return newStore(cfg) +} + +// SetLogger sets the logger used in the global store +func SetLogger(l nlogger.Logger) { + var c = instance() + c.cfg.Logger = l +} + +func (c *store) Name() string { + return c.name +} + +// RegisterLoader registers a Loader l with a given Watcher w. +func RegisterLoader(l Loader, loaderHooks ...func(Store) error) *ConfigLoader { + return instance().RegisterLoader(l, loaderHooks...) +} +func (c *store) RegisterLoader(l Loader, loaderHooks ...func(Store) error) *ConfigLoader { + var lw = c.newLoaderWatcher(l, NopWatcher{}, loaderHooks) + + c.WatcherLoaders = append( + c.WatcherLoaders, + lw, + ) + + return c.newConfigLoader(lw) +} + +// RegisterLoaderWatcher registers a WatcherLoader to the config. +func RegisterLoaderWatcher(lw LoaderWatcher, loaderHooks ...func(Store) error) *ConfigLoader { + return instance().RegisterLoaderWatcher(lw, loaderHooks...) +} +func (c *store) RegisterLoaderWatcher(lw LoaderWatcher, loaderHooks ...func(Store) error) *ConfigLoader { + var lwatcher = c.newLoaderWatcher(lw, lw, loaderHooks) + + c.WatcherClosers = append(c.WatcherClosers, lw) + c.WatcherLoaders = append( + c.WatcherLoaders, + lwatcher, + ) + + return c.newConfigLoader(lwatcher) +} + +// RegisterCloser adds a closer to the list of closers. +// Closers are closed when an error occured while reloading a config and the ExitOnError config is set to true +func RegisterCloser(closer io.Closer) Store { + return instance().RegisterCloser(closer) +} +func (c *store) RegisterCloser(closer io.Closer) Store { + c.Closers = append(c.Closers, closer) + return c +} + +// Instance returns the singleton global config store +func Instance() Store { + if c == nil { + c = newStore(DefaultConfig()) + } + return c +} + +// Stop stops the config store +func (c *store) stop() { + if err := c.WatcherClosers.Close(); err != nil { + c.cfg.Logger.Error(err.Error()) + } + + if err := c.Closers.Close(); err != nil { + c.cfg.Logger.Error(err.Error()) + } + + // exit on error unless specified + if !c.cfg.NoExitOnError { + os.Exit(c.cfg.ExitCode) + } +} + +func instance() *store { + if c == nil { + c = newStore(DefaultConfig()) + } + return c +} + +func reset() { + var cc = instance() + if cc != nil { + c = newStore(c.cfg) + } +} + +func newStore(cfg *Config) *store { + if cfg.Logger == nil { + cfg.Logger = defaultLogger() + } + + var mValue atomic.Value + var m = make(s) + mValue.Store(m) + + var s = &store{ + name: cfg.Name, + m: &mValue, + cfg: cfg, + mut: &sync.Mutex{}, + groups: make(map[string]*store), + WatcherLoaders: make([]*loaderWatcher, 0, 10), + WatcherClosers: make(Closers, 0, 10), + Closers: make(Closers, 0, 10), + } + + if s.name == "" { + s.name = defaultName + } + + // init metrics if it is enabled + if cfg.Metrics { + s.initMetrics() + } + + return s +} + +func defaultLogger() nlogger.Logger { + return nlogger.New(os.Stdout, "CONFIG | ") +} diff --git a/config_test.go b/config_test.go new file mode 100644 index 00000000..5fb2b19a --- /dev/null +++ b/config_test.go @@ -0,0 +1,377 @@ +package konfig + +import ( + "errors" + "log" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" +) + +type DummyLoader struct { + DataToLoad [][2]string + maxRetry int + retryDelay time.Duration + err bool +} + +func (d *DummyLoader) Load(s Values) error { + if d.err { + return errors.New("") + } + for _, dl := range d.DataToLoad { + log.Print("setting data", dl[0], dl[1]) + s.Set(dl[0], dl[1]) + } + log.Print("running loader") + return nil +} + +func (d *DummyLoader) Name() string { + return "dummy" +} + +func (d *DummyLoader) MaxRetry() int { + return d.maxRetry +} + +func (d *DummyLoader) RetryDelay() time.Duration { + return d.retryDelay +} + +func TestConfigWatcherLoader(t *testing.T) { + var testCases = []struct { + name string + setUp func(ctrl *gomock.Controller) + asserts func(t *testing.T) + loadErr bool + watchErr bool + }{ + { + name: "OneLoaderNoWatcher", + setUp: func(ctrl *gomock.Controller) { + RegisterLoader( + &DummyLoader{ + [][2]string{ + [2]string{ + "foo", "bar", + }, + }, + 1, + 3 * time.Second, + false, + }, + ) + }, + asserts: func(t *testing.T) { + require.Equal(t, "bar", MustString("foo")) + }, + }, + { + name: "OneLoaderNoWatcherOneError", + setUp: func(ctrl *gomock.Controller) { + + // set our expectations + var l = NewMockLoader(ctrl) + + l.EXPECT().MaxRetry().MinTimes(1).Return(2) + l.EXPECT().RetryDelay().MinTimes(1).Return(1 * time.Millisecond) + + gomock.InOrder( + l.EXPECT().Load(Values{}).Return(errors.New("")), + l.EXPECT().Load(Values{}).Return(nil), + ) + + RegisterLoader( + l, + ) + }, + asserts: func(t *testing.T) {}, + }, + + { + name: "OneLoaderNoWatcherErrorMaxRetry", + setUp: func(ctrl *gomock.Controller) { + + // set our expectations + var l = NewMockLoader(ctrl) + + l.EXPECT().MaxRetry().MinTimes(1).Return(2) + l.EXPECT().RetryDelay().MinTimes(1).Return(1 * time.Millisecond) + + gomock.InOrder( + l.EXPECT().Load(Values{}).Return(errors.New("")), + l.EXPECT().Load(Values{}).Return(errors.New("")), + l.EXPECT().Load(Values{}).Return(errors.New("")), + ) + + RegisterLoader( + l, + ) + }, + asserts: func(t *testing.T) {}, + loadErr: true, + }, + { + name: "OneWatcherLoaderError", + setUp: func(ctrl *gomock.Controller) { + // set our expectations + var wl = NewMockWatcher(ctrl) + var l = NewMockLoader(ctrl) + var c = make(chan struct{}, 1) + var d = make(chan struct{}) + + l.EXPECT().MaxRetry().MinTimes(1).Return(2) + l.EXPECT().RetryDelay().MinTimes(1).Return(1 * time.Millisecond) + + gomock.InOrder( + l.EXPECT().Load(Values{}).Return(nil), + l.EXPECT().Load(Values{}).Return(errors.New("")), + l.EXPECT().Load(Values{}).Return(errors.New("")), + l.EXPECT().Load(Values{}).Return(errors.New("")), + ) + wl.EXPECT().Start().Times(1).Return(nil) + wl.EXPECT().Done().MinTimes(1).Return(d) + wl.EXPECT().Watch().Return(c) + wl.EXPECT().Close().Return(nil) + // register the loader + RegisterLoaderWatcher( + NewLoaderWatcher( + l, + wl, + ), + ) + + // write to the watch chan + c <- struct{}{} + }, + asserts: func(t *testing.T) { + // we don't assert anything as we set expectations on the mock + // we make it wait long enough + time.Sleep(100 * time.Millisecond) + }, + }, + { + name: "MultiWatcherLoadersError", + setUp: func(ctrl *gomock.Controller) { + // set our expectations + var wl1 = NewMockWatcher(ctrl) + var l = NewMockLoader(ctrl) + var c = make(chan struct{}, 1) + var d = make(chan struct{}) + + var wl2 = NewMockWatcher(ctrl) + var l2 = NewMockLoader(ctrl) + var c2 = make(chan struct{}, 1) + var d2 = make(chan struct{}) + + l.EXPECT().MaxRetry().MinTimes(1).Return(2) + l.EXPECT().RetryDelay().MinTimes(1).Return(1 * time.Millisecond) + + l2.EXPECT().Load(Values{}).MinTimes(1).Return(nil) + + gomock.InOrder( + l.EXPECT().Load(Values{}).Return(nil), + l.EXPECT().Load(Values{}).Return(errors.New("")), + l.EXPECT().Load(Values{}).Return(errors.New("")), + l.EXPECT().Load(Values{}).Return(errors.New("")), + ) + wl2.EXPECT().Start().Times(1).Return(nil) + wl2.EXPECT().Done().MinTimes(1).Return(d2) + wl2.EXPECT().Watch().MinTimes(1).Return(c2) + wl2.EXPECT().Close().Times(1).Return(nil) + + wl1.EXPECT().Start().Times(1).Return(nil) + wl1.EXPECT().Done().MinTimes(1).Return(d) + wl1.EXPECT().Watch().Return(c) + wl1.EXPECT().Close().Return(nil) + + // register the loader + RegisterLoaderWatcher( + NewLoaderWatcher( + l, + wl1, + ), + func(Store) error { + return nil + }, + ) + RegisterLoaderWatcher( + NewLoaderWatcher( + l2, + wl2, + ), + func(Store) error { + return nil + }, + ) + + // close the watch chan so that it always goes through + close(c) + close(c2) + }, + asserts: func(t *testing.T) { + log.Print("sleeping") + time.Sleep(200 * time.Millisecond) + }, + }, + { + name: "MultiWatcherLoadersLoaderHooksError", + setUp: func(ctrl *gomock.Controller) { + // set our expectations + var wl1 = NewMockWatcher(ctrl) + var l = NewMockLoader(ctrl) + var c = make(chan struct{}, 1) + var d = make(chan struct{}) + + var wl2 = NewMockWatcher(ctrl) + var l2 = NewMockLoader(ctrl) + var c2 = make(chan struct{}, 1) + var d2 = make(chan struct{}) + + l2.EXPECT().Load(Values{}).MinTimes(1).Return(nil) + + gomock.InOrder( + l.EXPECT().Load(Values{}).Return(nil), + l.EXPECT().Load(Values{}).Return(nil), + ) + wl2.EXPECT().Start().Times(1).Return(nil) + wl2.EXPECT().Done().MinTimes(1).Return(d2) + wl2.EXPECT().Watch().MinTimes(1).Return(c2) + wl2.EXPECT().Close().Times(1).Return(nil) + + wl1.EXPECT().Start().Times(1).Return(nil) + wl1.EXPECT().Done().MinTimes(1).Return(d) + wl1.EXPECT().Watch().Return(c) + wl1.EXPECT().Close().Return(nil) + + var i int + // register the loader + RegisterLoaderWatcher( + NewLoaderWatcher( + l, + wl1, + ), + func(Store) error { + if i == 0 { + i++ + return nil + } + return errors.New("err") + }, + ) + RegisterLoaderWatcher( + NewLoaderWatcher( + l2, + wl2, + ), + func(Store) error { + return nil + }, + ) + + // close the watch chan so that it always goes through + close(c) + close(c2) + }, + asserts: func(t *testing.T) { + log.Print("sleeping") + time.Sleep(200 * time.Millisecond) + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + reset() + Init(DefaultConfig()) + var c = instance() + c.cfg.NoExitOnError = true + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + testCase.setUp(ctrl) + var err = Load() + if testCase.loadErr { + require.NotNil(t, err, "there should be an error") + return + } + require.Nil(t, err, "there should be no error") + err = Watch() + if testCase.watchErr { + require.Nil(t, err, "there should be no error") + } + testCase.asserts(t) + log.Print("test done") + }) + } + +} + +func TestRegisterLoaderHooks(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + reset() + var c = instance() + + var l = RegisterLoader( + NewMockLoader(ctrl), + ) + + require.Equal(t, 0, len(c.WatcherLoaders[0].loaderHooks)) + + l.AddHooks( + func(Store) error { return nil }, + ) + + require.Equal(t, 1, len(c.WatcherLoaders[0].loaderHooks)) +} + +type TestCloser struct { + err error + closed bool +} + +func (t *TestCloser) Close() error { + t.closed = true + return t.err +} + +func TestStop(t *testing.T) { + t.Run( + "no error, 2 closers", + func(t *testing.T) { + var c = New(DefaultConfig()).(*store) + var testCloser = &TestCloser{} + c.RegisterCloser(testCloser) + c.cfg.NoExitOnError = true + c.stop() + require.Equal(t, true, testCloser.closed) + }, + ) + + t.Run( + "no error, 2 closers", + func(t *testing.T) { + var c = New(DefaultConfig()) + var testCloser = &TestCloser{ + err: errors.New("foo"), + } + c.RegisterCloser(testCloser) + c.(*store).cfg.NoExitOnError = true + c.(*store).stop() + require.Equal(t, true, testCloser.closed) + }, + ) + reset() +} + +func TestGet(t *testing.T) { + reset() + Init(DefaultConfig()) + Set("FOO", "BAR") + require.Equal(t, "BAR", Get("FOO")) + require.Equal(t, nil, Get("IDONOTEXIST")) + reset() +} diff --git a/getter.go b/getter.go new file mode 100644 index 00000000..aab2d494 --- /dev/null +++ b/getter.go @@ -0,0 +1,14 @@ +package konfig + +import "github.com/lalamove/nui/ngetter" + +// Getter returns a mgetter.Getter for the key k +func Getter(k string) ngetter.GetterTyped { + return instance().Getter(k) +} + +func (c *store) Getter(k string) ngetter.GetterTyped { + return ngetter.GetterTypedFunc(func() interface{} { + return c.Get(k) + }) +} diff --git a/getter_test.go b/getter_test.go new file mode 100644 index 00000000..d96f69ef --- /dev/null +++ b/getter_test.go @@ -0,0 +1,23 @@ +package konfig + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetter(t *testing.T) { + t.Run( + "test new getter", + func(t *testing.T) { + Init(DefaultConfig()) + + var c = instance() + c.Set("int", 1) + + var g = Getter("int") + + require.Equal(t, "1", g.String()) + }, + ) +} diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..b0535ec0 --- /dev/null +++ b/go.mod @@ -0,0 +1,39 @@ +module github.com/lalamove/konfig + +require ( + github.com/lalamove/nui v0.0.0-20190108033743-206d8659d444 + github.com/BurntSushi/toml v0.3.1 + github.com/coreos/etcd v3.3.10+incompatible + github.com/davecgh/go-spew v1.1.1 + github.com/francoispqt/gojay v0.0.0-20181220093123-f2cc13a668ca + github.com/go-test/deep v1.0.1 + github.com/gogo/protobuf v1.2.0 + github.com/golang/mock v1.2.0 + github.com/golang/protobuf v1.2.0 + github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db + github.com/hashicorp/errwrap v1.0.0 + github.com/hashicorp/go-cleanhttp v0.5.0 + github.com/hashicorp/go-multierror v1.0.0 + github.com/hashicorp/go-retryablehttp v0.5.0 + github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90 + github.com/hashicorp/go-sockaddr v0.0.0-20190103214136-e92cdb5343bb + github.com/hashicorp/hcl v1.0.0 + github.com/hashicorp/vault v1.0.1 + github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3 + github.com/mitchellh/go-homedir v1.0.0 + github.com/mitchellh/mapstructure v1.1.2 + github.com/pierrec/lz4 v0.0.0-20181005164709-635575b42742 + github.com/pmezard/go-difflib v1.0.0 + github.com/radovskyb/watcher v1.0.5 + github.com/ryanuber/go-glob v0.0.0-20160226084822-572520ed46db + github.com/spf13/cast v1.3.0 + github.com/stretchr/testify v1.3.0 + go.etcd.io/etcd v3.3.10+incompatible + golang.org/x/net v0.0.0-20190110200230-915654e7eabc + golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb + golang.org/x/text v0.3.0 + golang.org/x/time v0.0.0-20181108054448-85acf8d2951c + google.golang.org/genproto v0.0.0-20190110221437-6909d8a4a91b + google.golang.org/grpc v1.17.0 + gopkg.in/yaml.v2 v2.2.2 +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..dc4ae905 --- /dev/null +++ b/go.sum @@ -0,0 +1,88 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/francoispqt/gojay v0.0.0-20181220093123-f2cc13a668ca h1:F2BD6Vhei4w0rtm4eNpzylNsB07CcCbpYA+xlqMx3mA= +github.com/francoispqt/gojay v0.0.0-20181220093123-f2cc13a668ca/go.mod h1:H8Wgri1Asi1VevY3ySdpIK5+KCpqzToVswNq8g2xZj4= +github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg= +github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-retryablehttp v0.5.0 h1:aVN0FYnPwAgZI/hVzqwfMiM86ttcHTlQKbBVeVmXPIs= +github.com/hashicorp/go-retryablehttp v0.5.0/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90 h1:VBj0QYQ0u2MCJzBfeYXGexnAl17GsH1yidnoxCqqD9E= +github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90/go.mod h1:o4zcYY1e0GEZI6eSEr+43QDYmuGglw1qSO6qdHUHCgg= +github.com/hashicorp/go-sockaddr v0.0.0-20190103214136-e92cdb5343bb h1:YrwA8w5SBkUIH5BzN2pMYhno+txUCOD5+PVXwLS6ddI= +github.com/hashicorp/go-sockaddr v0.0.0-20190103214136-e92cdb5343bb/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/vault v1.0.1 h1:x3hcjkJLd5L4ehPhZcraokFO7dq8MJ3oKvQtrkIiIU8= +github.com/hashicorp/vault v1.0.1/go.mod h1:KfSyffbKxoVyspOdlaGVjIuwLobi07qD1bAbosPMpP0= +github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3 h1:sHsPfNMAG70QAvKbddQ0uScZCHQoZsT5NykGRCeeeIs= +github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/lalamove/nui v0.0.0-20190108033743-206d8659d444 h1:W6TG7IqV3wxq7XhaUL21FPO2zdiRAl94KP/glqudDoI= +github.com/lalamove/nui v0.0.0-20190108033743-206d8659d444/go.mod h1:OopVZQOPFepLy2qh/ASzzQCcijBf6bNPO3369NtUGV8= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/pierrec/lz4 v0.0.0-20181005164709-635575b42742 h1:wKfigKMTgvSzBLIVvB5QaBBQI0odU6n45/UKSphjLus= +github.com/pierrec/lz4 v0.0.0-20181005164709-635575b42742/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/radovskyb/watcher v1.0.5 h1:wqt7gb+HjGacvFoLTKeT44C+XVPxu7bvHvKT1IvZ7rw= +github.com/radovskyb/watcher v1.0.5/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg= +github.com/ryanuber/go-glob v0.0.0-20160226084822-572520ed46db h1:ge9atzKq16843f793fDVxKUhmTb4H5muzjJQ6PgsnHg= +github.com/ryanuber/go-glob v0.0.0-20160226084822-572520ed46db/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +go.etcd.io/etcd v3.3.10+incompatible h1:qXVcIR1kU3CYLD8zXDseOmBNwg0uaui53e4Wg4uj0rk= +go.etcd.io/etcd v3.3.10+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190110200230-915654e7eabc h1:Yx9JGxI1SBhVLFjpAkWMaO1TF+xyqtHLjZpvQboJGiM= +golang.org/x/net v0.0.0-20190110200230-915654e7eabc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb h1:1w588/yEchbPNpa9sEvOcMZYbWHedwJjg4VOAdDHWHk= +golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190110221437-6909d8a4a91b h1:oNKY5TkqnRYR7KgHuuuDi9g7tZmgWDNx0FFRv4u0j9A= +google.golang.org/genproto v0.0.0-20190110221437-6909d8a4a91b/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/group.go b/group.go new file mode 100644 index 00000000..52adefcf --- /dev/null +++ b/group.go @@ -0,0 +1,23 @@ +package konfig + +// Group gets a group of configs +func Group(groupName string) Store { + return instance().Group(groupName) +} + +// Group gets a group of configs +func (c *store) Group(groupName string) Store { + return c.lazyGroup(groupName) +} + +func (c *store) lazyGroup(groupName string) Store { + c.mut.Lock() + defer c.mut.Unlock() + if v, ok := c.groups[groupName]; ok { + return v + } + c.groups[groupName] = newStore(c.cfg) + c.groups[groupName].name = groupName + + return c.groups[groupName] +} diff --git a/group_test.go b/group_test.go new file mode 100644 index 00000000..7dbdcccb --- /dev/null +++ b/group_test.go @@ -0,0 +1,24 @@ +package konfig + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGroup(t *testing.T) { + t.Run( + "test basic group", + func(t *testing.T) { + var c = newStore(DefaultConfig()) + var g = c.Group("test") + + c.Set("foo", "bar") + require.Equal(t, nil, g.Get("foo")) + + g.Set("foo", "bar") + var gg = c.Group("test") + require.Equal(t, "bar", gg.Get("foo")) + }, + ) +} diff --git a/loader.go b/loader.go new file mode 100644 index 00000000..469f4947 --- /dev/null +++ b/loader.go @@ -0,0 +1,226 @@ +package konfig + +import ( + "errors" + "fmt" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +// ErrNoLoaders is the error returned when no loaders are set in the config and Load is called +var ErrNoLoaders = errors.New("No loaders in config") + +// Values is the values attached to a loader +type Values map[string]interface{} + +// Set adds a key value to the Values +func (x Values) Set(k string, v interface{}) { + x[k] = v +} + +func (x Values) load(ox Values, c *store) { + c.mut.Lock() + defer c.mut.Unlock() + + var m = c.m.Load().(s) + + // we copy the previous store + // but we omit what was on the previous values + var nm = make(s) + for kk, vv := range m { + if _, ok := ox[kk]; !ok { + nm[kk] = vv + } + } + // we add the new values + for kk, vv := range x { + nm[kk] = vv + } + + // if there is a value bound we set it there also + if c.v != nil { + c.v.setValues(ox, x) + } + + c.m.Store(nm) +} + +// Loader is the interface a config loader must implement to be used withint the package +type Loader interface { + // Name returns the name of the loader + Name() string + // Load loads config values in a Values + Load(Values) error + // MaxRetry returns the max number of times to retry when Load fails + MaxRetry() int + // RetryDelay returns the delay between each retry + RetryDelay() time.Duration +} + +// LoaderHooks are functions ran when a config load has been performed +type LoaderHooks []func(Store) error + +// Run runs all loader and stops when it encounters an error +func (l LoaderHooks) Run(cfg Store) error { + for _, h := range l { + if err := h(cfg); err != nil { + return err + } + } + return nil +} + +// LoadWatch loads the config then starts watching it +func LoadWatch() error { + return instance().LoadWatch() +} +func (c *store) LoadWatch() error { + if err := c.Load(); err != nil { + return err + } else if err := c.Watch(); err != nil { + return err + } + return nil +} + +// Load is a function running load on the global config instance +func Load() error { + return instance().Load() +} + +func (c *store) Load() error { + if len(c.WatcherLoaders) == 0 { + panic(ErrNoLoaders) + } + for _, l := range c.WatcherLoaders { + // we load the loader once, then we start the reload worker with the watcher + if err := c.loaderLoadRetry(l, 0); err != nil { + c.stop() + return err + } + } + return nil +} + +// ConfigLoader is a wrapper of Loader with methods to add hooks +type ConfigLoader struct { + *loaderWatcher + mut *sync.Mutex +} + +func (c *store) newConfigLoader(lw *loaderWatcher) *ConfigLoader { + var cl = &ConfigLoader{ + loaderWatcher: lw, + mut: &sync.Mutex{}, + } + + return cl +} + +// AddHooks adds hooks to the loader +func (cl *ConfigLoader) AddHooks(loaderHooks ...func(Store) error) *ConfigLoader { + cl.mut.Lock() + defer cl.mut.Unlock() + + if cl.loaderWatcher.loaderHooks == nil { + cl.loaderWatcher.loaderHooks = make(LoaderHooks, 0) + } + + cl.loaderWatcher.loaderHooks = append( + cl.loaderWatcher.loaderHooks, + loaderHooks..., + ) + + return cl +} + +// We don't look for Done on the watcher here as the NopWatcher needs to run load at least once +func (c *store) loaderLoadRetry(wl *loaderWatcher, retry int) error { + // we create a new Values + var v = make(Values, len(wl.values)) + + // we call the loader + if err := wl.Load(v); err != nil { + time.Sleep(wl.RetryDelay()) + if retry >= wl.MaxRetry() { + return err + } + return c.loaderLoadRetry(wl, retry+1) + } + + // we add the values to the store + v.load(wl.values, c) + wl.values = v + + // we run the hooks + if wl.loaderHooks != nil { + c.mut.Lock() + if err := wl.loaderHooks.Run(c); err != nil { + c.cfg.Logger.Error("Error while running loader hooks: " + err.Error()) + c.mut.Unlock() + return err + } + c.mut.Unlock() + } + + return nil +} + +func (c *store) watchLoader(wl *loaderWatcher) { + // if a panic occurs close everything + defer func() { + if r := recover(); r != nil { + c.cfg.Logger.Error(fmt.Sprintf("%v", r)) + c.stop() + return + } + }() + + // make sure we recover from panics + for { + select { + case <-wl.Done(): + if err := wl.Err(); err != nil { + c.cfg.Logger.Error(err.Error()) + } + // the watcher is closed + return + case <-wl.Watch(): + // we got an event + // do a loaderLoadRetry + select { + case <-wl.Done(): + if err := wl.Err(); err != nil { + c.cfg.Logger.Error(err.Error()) + } + return + default: + + var t *prometheus.Timer + if c.cfg.Metrics { + t = prometheus.NewTimer(wl.metrics.configReloadDuration) + } + + if err := c.loaderLoadRetry(wl, 0); err != nil { + // if metrics is enabled we record a load failure + if c.cfg.Metrics { + wl.metrics.configReloadFailure.Inc() + t.ObserveDuration() + } + + if !c.cfg.NoStopOnFailure { + c.stop() + } + return + } + + if c.cfg.Metrics { + t.ObserveDuration() + wl.metrics.configReloadSuccess.Inc() + } + } + } + } +} diff --git a/loader/klenv/README.md b/loader/klenv/README.md new file mode 100644 index 00000000..24634a59 --- /dev/null +++ b/loader/klenv/README.md @@ -0,0 +1,34 @@ +# Env Loader +Env loader loads environment variables in a konfig.Store + +# Usage + +Basic usage loading all environment variables +```go +envLoader := klenv.New(&klenv.Config{}) +``` + +Loading specific variables +```go +envLoader := klenv.New(&klenv.Config{ + Vars: []string{ + "DEBUG", + "PORT", + }, +}) +``` + +Loading specific variables if key matches regexp +```go +envLoader := klenv.New(&klenv.Config{ + Regexp: "^APP_.*" +}) +``` + +With a replacer and a Prefix for keys +```go +envLoader := klenv.New(&klenv.Config{ + Prefix: "config.", + Replacer: nstrings.ReplacerToLower, +}) +``` diff --git a/loader/klenv/envloader.go b/loader/klenv/envloader.go new file mode 100644 index 00000000..baeef10f --- /dev/null +++ b/loader/klenv/envloader.go @@ -0,0 +1,109 @@ +package klenv + +import ( + "os" + "regexp" + "strings" + "time" + + "github.com/lalamove/konfig" + "github.com/lalamove/nui/nstrings" +) + +var ( + _ konfig.Loader = (*Loader)(nil) +) + +const ( + sepEnvVar = "=" + defaultName = "env" +) + +// Config is the config a an EnvLoader +type Config struct { + // Name is the name of the loader + Name string + // Regexp will load the environment variable if it matches the given regexp + Regexp string + // Vars will load vars only present in the vars slice + Vars []string + // Prefix will add a prefix to the environment variables when adding them in the config store + Prefix string + // Replacer is used to replace chars in env vars keys + Replacer nstrings.Replacer + // MaxRetry is the maximum number of time the load method can be retried when it fails + MaxRetry int + // RetryDelay is the time betweel each retry + RetryDelay time.Duration +} + +// Loader is the structure representing the environment loader +type Loader struct { + cfg *Config + r *regexp.Regexp +} + +// New return a new environment loader with the given config +func New(cfg *Config) *Loader { + var r *regexp.Regexp + if cfg.Regexp != "" { + r = regexp.MustCompile(cfg.Regexp) + } + + if cfg.Name == "" { + cfg.Name = defaultName + } + + return &Loader{ + cfg, + r, + } +} + +// Name returns the name of the loader +func (l *Loader) Name() string { return l.cfg.Name } + +// Load implements konfig.Loader, it loads environment variables into the konfig.Store +// based on config passed to the loader +func (l *Loader) Load(s konfig.Values) error { + if l.cfg.Vars != nil && len(l.cfg.Vars) > 0 { + return l.loadVars(s) + } + for _, v := range os.Environ() { + var spl = strings.SplitN(v, sepEnvVar, 2) + // if has regex and key does not macth regexp we continue + if l.r != nil && !l.r.MatchString(spl[0]) { + continue + } + var k = spl[0] + if l.cfg.Replacer != nil { + k = l.cfg.Replacer.Replace(k) + } + k = l.cfg.Prefix + k + s.Set(k, spl[1]) + } + + return nil +} + +// MaxRetry returns the maximum number to retry a load when an error occurs +func (l *Loader) MaxRetry() int { + return l.cfg.MaxRetry +} + +// RetryDelay returns the delay between each load retry +func (l *Loader) RetryDelay() time.Duration { + return l.cfg.RetryDelay +} + +func (l *Loader) loadVars(s konfig.Values) error { + for _, k := range l.cfg.Vars { + var v = os.Getenv(k) + if l.cfg.Replacer != nil { + k = l.cfg.Replacer.Replace(k) + } + k = l.cfg.Prefix + k + s.Set(k, v) + } + return nil +} diff --git a/loader/klenv/envloader_test.go b/loader/klenv/envloader_test.go new file mode 100644 index 00000000..0cede24b --- /dev/null +++ b/loader/klenv/envloader_test.go @@ -0,0 +1,132 @@ +package klenv + +import ( + "os" + "testing" + "time" + + "github.com/lalamove/konfig" + "github.com/lalamove/nui/nstrings" + "github.com/stretchr/testify/require" +) + +func TestEnvLoader(t *testing.T) { + t.Run( + "load defined env vars", + func(t *testing.T) { + os.Setenv("FOO", "BAR") + os.Setenv("BAR", "FOO") + + var l = New(&Config{ + Vars: []string{ + "FOO", + "BAR", + }, + }) + + var v = konfig.Values{} + l.Load(v) + + require.Equal(t, "BAR", v["FOO"]) + require.Equal(t, "FOO", v["BAR"]) + }, + ) + + t.Run( + "load env vars regexp", + func(t *testing.T) { + konfig.Init(konfig.DefaultConfig()) + + os.Setenv("FOO", "BAR") + os.Setenv("BAR", "FOO") + + var l = New(&Config{ + Regexp: "^F.*", + }) + + var v = konfig.Values{} + l.Load(v) + + require.Equal(t, "BAR", v["FOO"]) + var _, ok = v["BAR"] + require.Equal(t, false, ok) + }, + ) + + t.Run( + "load env vars prefix regexp", + func(t *testing.T) { + konfig.Init(konfig.DefaultConfig()) + + os.Setenv("FOO", "BAR") + os.Setenv("BAR", "FOO") + + var l = New(&Config{ + Regexp: "^F.*", + Prefix: "KONFIG_", + }) + + var v = konfig.Values{} + l.Load(v) + + require.Equal(t, "BAR", v["KONFIG_FOO"]) + + var _, ok = v["KONFIG_BAR"] + require.Equal(t, false, ok) + }, + ) + + t.Run( + "load env vars prefix regexp replacer", + func(t *testing.T) { + konfig.Init(konfig.DefaultConfig()) + + os.Setenv("FOO", "BAR") + os.Setenv("BAR", "FOO") + + var l = New(&Config{ + Regexp: "^F.*", + Prefix: "KONFIG_", + Replacer: nstrings.ReplacerToLower, + }) + + var v = konfig.Values{} + l.Load(v) + + require.Equal(t, "BAR", v["KONFIG_foo"]) + + var _, ok = v["KONFIG_bar"] + require.Equal(t, false, ok) + }, + ) + + t.Run( + "new loader invalid regexp", + func(t *testing.T) { + konfig.Init(konfig.DefaultConfig()) + + os.Setenv("FOO", "BAR") + os.Setenv("BAR", "FOO") + + require.Panics(t, func() { + New(&Config{ + Regexp: "[", + }) + }) + + }, + ) + + t.Run( + "test max retry", + func(t *testing.T) { + var l = New(&Config{ + MaxRetry: 1, + RetryDelay: 1 * time.Second, + }) + + require.Equal(t, 1, l.MaxRetry()) + require.Equal(t, 1*time.Second, l.RetryDelay()) + }, + ) +} diff --git a/loader/kletcd/README.md b/loader/kletcd/README.md new file mode 100644 index 00000000..fc7198ec --- /dev/null +++ b/loader/kletcd/README.md @@ -0,0 +1,31 @@ +# Etcd Loader +Loads configs from Etcd into konfig.Store + +# Usage + +Basic usage loading keys and using result as string with watcher +```go +etcdLoader := kletcd.New(&kletc.Config{ + Client: etcdClient, // from go.etcd.io/etcd/clientv3 package + Keys: []Key{ + { + Key: "foo/bar", + }, + }, + Watch: true, +}) +``` + +Loading keys and JSON parser +```go +etcdLoader := kletcd.New(&kletc.Config{ + Client: etcdClient, // from go.etcd.io/etcd/clientv3 package + Keys: []Key{ + { + Key: "foo/bar", + Parser: kpjson.Parser, + }, + }, + Watch: true, +}) +``` diff --git a/loader/kletcd/etcdloader.go b/loader/kletcd/etcdloader.go new file mode 100644 index 00000000..66d97e6b --- /dev/null +++ b/loader/kletcd/etcdloader.go @@ -0,0 +1,179 @@ +package kletcd + +import ( + "bytes" + "context" + "time" + + "github.com/coreos/etcd/mvcc/mvccpb" + "github.com/lalamove/konfig" + "github.com/lalamove/konfig/parser" + "github.com/lalamove/konfig/watcher/kwpoll" + "github.com/lalamove/nui/ncontext" + "github.com/lalamove/nui/nstrings" + "go.etcd.io/etcd/clientv3" +) + +var ( + defaultTimeout = 5 * time.Second + _ konfig.Loader = (*Loader)(nil) +) + +const ( + defaultName = "etcd" +) + +// Key is an Etcd Key to load +type Key struct { + // Key is the etcd key + Key string + // Parser is the parser for the key + // If nil, the value is casted to a string before adding to the config.Store + Parser parser.Parser +} + +// Config is the structure representing the config of a Loader +type Config struct { + // Name is the name of the loader + Name string + // Client is the etcd KV client + Client clientv3.KV + // Keys is the list of keys to fetch + Keys []Key + // Timeout is the timeout duration when fetching a key + Timeout time.Duration + // Prefix is a prefix to prepend keys when adding into the konfig.Store + Prefix string + // Replacer is a Replacer for the key before adding to the konfig.Store + Replacer nstrings.Replacer + // Watch tells wether there should be a watcher with the loader + Watch bool + // Rater is the rater to pass to the poll watcher + Rater kwpoll.Rater + // MaxRetry is the maximum number of times we can retry to load if it fails + MaxRetry int + // RetryDelay is the time between each retry when a load fails + RetryDelay time.Duration + + contexter ncontext.Contexter +} + +// Loader is the structure of a loader +type Loader struct { + *kwpoll.PollWatcher + cfg *Config +} + +// New returns a new loader with the given config +func New(cfg *Config) *Loader { + if cfg.Timeout == 0 { + cfg.Timeout = defaultTimeout + } + + if cfg.contexter == nil { + cfg.contexter = ncontext.DefaultContexter + } + + if cfg.Name == "" { + cfg.Name = defaultName + } + + var l = &Loader{ + cfg: cfg, + } + + if cfg.Watch { + var r, err = l.Get() + if err != nil { + panic(err) + } + l.PollWatcher = kwpoll.New(&kwpoll.Config{ + Getter: l, + Rater: cfg.Rater, + InitValue: r, + }) + } + + return l +} + +// Name returns the name of the loader +func (l *Loader) Name() string { return l.cfg.Name } + +// Load loads the values from the keys defined by the config in the konfig.Store +func (l *Loader) Load(s konfig.Values) error { + for _, k := range l.cfg.Keys { + + values, err := l.keyValue(k.Key) + if err != nil { + return err + } + + for _, v := range values { + var configKey = l.cfg.Prefix + string(v.Key) + if l.cfg.Replacer != nil { + configKey = l.cfg.Replacer.Replace(configKey) + } + + // if the key has a parser, we parse the key value using the provided Parser + // else we just convert the value to a string + if k.Parser != nil { + if err := k.Parser.Parse(bytes.NewReader(v.Value), s); err != nil { + return err + } + } else { + s.Set(configKey, string(v.Value)) + } + } + } + + return nil +} + +// Get implements kwpoll.Getter, it returns a representation of the keys and their values +func (l *Loader) Get() (interface{}, error) { + var result = make(map[string]map[string][]byte) + for _, k := range l.cfg.Keys { + + values, err := l.keyValue(k.Key) + if err != nil { + return nil, err + } + + result[k.Key] = make(map[string][]byte) + + for _, v := range values { + var configKey = l.cfg.Prefix + string(v.Key) + if l.cfg.Replacer != nil { + configKey = l.cfg.Replacer.Replace(configKey) + } + result[k.Key][configKey] = v.Value + } + } + return result, nil +} + +// MaxRetry is the maximum number of time to retry when a load fails +func (l *Loader) MaxRetry() int { + return l.cfg.MaxRetry +} + +// RetryDelay is the delay between each retry +func (l *Loader) RetryDelay() time.Duration { + return l.cfg.RetryDelay +} + +func (l *Loader) keyValue(k string) ([]*mvccpb.KeyValue, error) { + var ctx, cancel = l.cfg.contexter.WithTimeout( + context.Background(), + l.cfg.Timeout, + ) + defer cancel() + + values, err := l.cfg.Client.Get(ctx, k) + if err != nil { + return nil, err + } + + return values.Kvs, nil +} diff --git a/loader/kletcd/etcdloader_test.go b/loader/kletcd/etcdloader_test.go new file mode 100644 index 00000000..b6dd2e04 --- /dev/null +++ b/loader/kletcd/etcdloader_test.go @@ -0,0 +1,382 @@ +package kletcd + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/lalamove/konfig" + "github.com/lalamove/konfig/mocks" + "github.com/lalamove/konfig/watcher/kwpoll" + "github.com/coreos/etcd/mvcc/mvccpb" + gomock "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + "go.etcd.io/etcd/clientv3" +) + +func TestEtcdLoader(t *testing.T) { + t.Run( + "basic no error", + func(t *testing.T) { + konfig.Init(konfig.DefaultConfig()) + + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + var mockClient = mocks.NewMockKV(ctrl) + var mockContexter = mocks.NewMockContexter(ctrl) + + var ctx, _ = context.WithTimeout( + context.Background(), + 5*time.Second, + ) + + mockContexter.EXPECT().WithTimeout( + context.Background(), + 5*time.Second, + ).Times(2).Return(ctx, context.CancelFunc(func() {})) + + mockClient.EXPECT().Get(ctx, "key1").Return( + &clientv3.GetResponse{ + Kvs: []*mvccpb.KeyValue{ + &mvccpb.KeyValue{ + Key: []byte(`key1`), + Value: []byte(`bar`), + }, + }, + }, + nil, + ) + + mockClient.EXPECT().Get(ctx, "key2").Return( + &clientv3.GetResponse{ + Kvs: []*mvccpb.KeyValue{ + &mvccpb.KeyValue{ + Key: []byte(`key2`), + Value: []byte(`foo`), + }, + }, + }, + nil, + ) + + var l = New(&Config{ + Client: mockClient, + Keys: []Key{ + {Key: "key1"}, + {Key: "key2"}, + }, + }) + + l.cfg.contexter = mockContexter + + var v = konfig.Values{} + + l.Load(v) + + require.Equal(t, "bar", v["key1"]) + require.Equal(t, "foo", v["key2"]) + }, + ) + + t.Run( + "basic no error multiple result in a key", + func(t *testing.T) { + konfig.Init(konfig.DefaultConfig()) + + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + var mockClient = mocks.NewMockKV(ctrl) + var mockContexter = mocks.NewMockContexter(ctrl) + + var ctx, _ = context.WithTimeout( + context.Background(), + 5*time.Second, + ) + + mockContexter.EXPECT(). + WithTimeout( + context.Background(), + 5*time.Second, + ). + Times(1). + Return(ctx, context.CancelFunc(func() {})) + + mockClient.EXPECT().Get(ctx, "key1").Return( + &clientv3.GetResponse{ + Kvs: []*mvccpb.KeyValue{ + &mvccpb.KeyValue{ + Key: []byte(`key1`), + Value: []byte(`bar`), + }, + &mvccpb.KeyValue{ + Key: []byte(`key2`), + Value: []byte(`foo`), + }, + }, + }, + nil, + ) + + var l = New(&Config{ + Client: mockClient, + Keys: []Key{ + {Key: "key1"}, + }, + }) + + l.cfg.contexter = mockContexter + + var v = konfig.Values{} + + l.Load(v) + + require.Equal(t, "bar", v["key1"]) + require.Equal(t, "foo", v["key2"]) + }, + ) + + t.Run( + "no error multiple result in a key replacer prefix", + func(t *testing.T) { + konfig.Init(konfig.DefaultConfig()) + + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + var mockClient = mocks.NewMockKV(ctrl) + var mockContexter = mocks.NewMockContexter(ctrl) + + var ctx, _ = context.WithTimeout( + context.Background(), + 5*time.Second, + ) + + mockContexter.EXPECT(). + WithTimeout( + context.Background(), + 5*time.Second, + ). + Times(1). + Return(ctx, context.CancelFunc(func() {})) + + mockClient.EXPECT().Get(ctx, "key1").Return( + &clientv3.GetResponse{ + Kvs: []*mvccpb.KeyValue{ + &mvccpb.KeyValue{ + Key: []byte(`key1`), + Value: []byte(`bar`), + }, + &mvccpb.KeyValue{ + Key: []byte(`key2`), + Value: []byte(`foo`), + }, + }, + }, + nil, + ) + + var l = New(&Config{ + Client: mockClient, + Keys: []Key{ + {Key: "key1"}, + }, + Prefix: "pfx_", + Replacer: strings.NewReplacer("key", "yek"), + }) + + l.cfg.contexter = mockContexter + + var v = konfig.Values{} + l.Load(v) + + require.Equal(t, "bar", v["pfx_yek1"]) + require.Equal(t, "foo", v["pfx_yek2"]) + }, + ) + + t.Run( + "no error multiple result in a key replacer prefix", + func(t *testing.T) { + konfig.Init(konfig.DefaultConfig()) + + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + var mockClient = mocks.NewMockKV(ctrl) + var mockContexter = mocks.NewMockContexter(ctrl) + + var ctx, _ = context.WithTimeout( + context.Background(), + 5*time.Second, + ) + + mockContexter.EXPECT(). + WithTimeout( + context.Background(), + 5*time.Second, + ). + Times(1). + Return(ctx, context.CancelFunc(func() {})) + + mockClient.EXPECT().Get(ctx, "key1").Return( + nil, + errors.New(""), + ) + + var l = New(&Config{ + Client: mockClient, + Keys: []Key{{Key: "key1"}}, + Prefix: "pfx_", + Replacer: strings.NewReplacer("key", "yek"), + }) + + l.cfg.contexter = mockContexter + + err := l.Load(konfig.Values{}) + + require.NotNil(t, err) + }, + ) + +} + +func TestGetter(t *testing.T) { + t.Run( + "multiple keys no error", + func(t *testing.T) { + konfig.Init(konfig.DefaultConfig()) + + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + var mockClient = mocks.NewMockKV(ctrl) + var mockContexter = mocks.NewMockContexter(ctrl) + + var ctx, _ = context.WithTimeout( + context.Background(), + 5*time.Second, + ) + + mockContexter.EXPECT(). + WithTimeout( + context.Background(), + 5*time.Second, + ). + Times(1). + Return(ctx, context.CancelFunc(func() {})) + + mockClient.EXPECT().Get(ctx, "key1").Return( + &clientv3.GetResponse{ + Kvs: []*mvccpb.KeyValue{ + &mvccpb.KeyValue{ + Key: []byte(`key1`), + Value: []byte(`bar`), + }, + &mvccpb.KeyValue{ + Key: []byte(`key2`), + Value: []byte(`foo`), + }, + }, + }, + nil, + ) + + var l = New(&Config{ + Client: mockClient, + Keys: []Key{{Key: "key1"}}, + Prefix: "pfx_", + Replacer: strings.NewReplacer("key", "yek"), + }) + + l.cfg.contexter = mockContexter + + r, err := l.Get() + + require.Nil(t, err) + + require.Equal( + t, + map[string]map[string][]byte{ + "key1": map[string][]byte{ + "pfx_yek1": []byte("bar"), + "pfx_yek2": []byte("foo"), + }, + }, + r, + ) + }, + ) + + t.Run( + "loader with watcher and parser on keys", + func(t *testing.T) { + + konfig.Init(konfig.DefaultConfig()) + + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + var mockClient = mocks.NewMockKV(ctrl) + var mockContexter = mocks.NewMockContexter(ctrl) + + var ctx, _ = context.WithTimeout( + context.Background(), + 5*time.Second, + ) + + mockContexter.EXPECT(). + WithTimeout( + context.Background(), + 5*time.Second, + ). + MinTimes(1). + Return(ctx, context.CancelFunc(func() {})) + + mockClient.EXPECT().Get(ctx, "key1").Times(2).Return( + &clientv3.GetResponse{ + Kvs: []*mvccpb.KeyValue{ + &mvccpb.KeyValue{ + Key: []byte(`key1`), + Value: []byte(`bar`), + }, + &mvccpb.KeyValue{ + Key: []byte(`key2`), + Value: []byte(`foo`), + }, + }, + }, + nil, + ) + + var l = New(&Config{ + Client: mockClient, + Keys: []Key{{Key: "key1"}}, + Prefix: "pfx_", + Replacer: strings.NewReplacer("key", "yek"), + Watch: true, + Rater: kwpoll.Time(1 * time.Second), + contexter: mockContexter, + }) + + r, err := l.Get() + + require.Nil(t, err) + + require.Equal( + t, + map[string]map[string][]byte{ + "key1": map[string][]byte{ + "pfx_yek1": []byte("bar"), + "pfx_yek2": []byte("foo"), + }, + }, + r, + ) + + }, + ) +} diff --git a/loader/klfile/README.md b/loader/klfile/README.md new file mode 100644 index 00000000..69e76bf1 --- /dev/null +++ b/loader/klfile/README.md @@ -0,0 +1,25 @@ +# File Loader +File loader loads config from files + +# Usage + +Basic usage with files and json parser and a watcher +```go +fileLoader := klfile.New(&klfile.Config{ + Files: []File{ + { + Path: "./config.json", + Parser: kpjson.Parser, + }, + }, + Watch: true, + Rate: 1 * time.Second, // Rate for the polling watching the file changes +}) +``` + +Simplified syntax: +```go +fileLoader := klfile. + NewFileLoader("config-files", kpjson.Parser, "file1.json", "file2.json"). + WithWatcher() +``` diff --git a/loader/klfile/fileloader.go b/loader/klfile/fileloader.go new file mode 100644 index 00000000..fad13ec0 --- /dev/null +++ b/loader/klfile/fileloader.go @@ -0,0 +1,178 @@ +package klfile + +import ( + "errors" + "os" + "time" + + "github.com/lalamove/konfig" + "github.com/lalamove/konfig/parser" + "github.com/lalamove/konfig/watcher/kwfile" + "github.com/lalamove/nui/nfs" + "github.com/lalamove/nui/nlogger" +) + +var ( + _ konfig.Loader = (*Loader)(nil) + // ErrNoFiles is the error thrown when trying to create a file loader with no files in config + ErrNoFiles = errors.New("no files provided") + // ErrNoParser is the error thrown when trying to create a file loader with no parser + ErrNoParser = errors.New("no parser provided") + // DefaultRate is the default polling rate to check files + DefaultRate = 10 * time.Second +) + +const ( + defaultName = "file" +) + +// File is a file to load from +type File struct { + // Path is the path to the file + Path string + // Parser is the parser used to parse file and add it to the config store + Parser parser.Parser +} + +// Config is the config for the file loader +type Config struct { + // Name is the name of the loader + Name string + // Files is the path to the files to load + Files []File + // MaxRetry is the maximum number of times load can be retried in config + MaxRetry int + // RetryDelay is the delay between each retry + RetryDelay time.Duration + // Debug sets the debug mode on the file loader + Debug bool + // Logger is the logger used to print messages + Logger nlogger.Logger + // Watch sets whether the fileloader should also watch be a konfig.Watcher + Watch bool + // Rate is the kwfile polling rate + // Default is 10 seconds + Rate time.Duration +} + +// Loader is the structure representring a file loader. +// A file loader loads data from a file and stores it in the konfig.Store. +type Loader struct { + *kwfile.FileWatcher + cfg *Config + fs nfs.FileSystem +} + +// New creates a new Loader fromt the Config cfg. +func New(cfg *Config) *Loader { + if cfg.Files == nil || len(cfg.Files) == 0 { + panic(ErrNoFiles) + } + // make sure all files have a parser + for _, f := range cfg.Files { + if f.Parser == nil { + panic(ErrNoParser) + } + } + if cfg.Logger == nil { + cfg.Logger = defaultLogger() + } + + if cfg.Name == "" { + cfg.Name = defaultName + } + + // create the watcher + var fw *kwfile.FileWatcher + if cfg.Watch { + var filePaths = make([]string, len(cfg.Files)) + for i, f := range cfg.Files { + filePaths[i] = f.Path + } + fw = kwfile.New( + &kwfile.Config{ + Files: filePaths, + Rate: cfg.Rate, + Debug: cfg.Debug, + Logger: cfg.Logger, + }, + ) + } + + return &Loader{ + FileWatcher: fw, + cfg: cfg, + fs: nfs.OSFileSystem{}, + } +} + +// NewFileLoader returns a new file loader with the given name n, the parser p and the file paths filePaths +func NewFileLoader(n string, p parser.Parser, filePaths ...string) *Loader { + var files = make([]File, len(filePaths)) + for i, fp := range filePaths { + files[i] = File{ + Path: fp, + Parser: p, + } + } + + return New(&Config{ + Name: n, + Files: files, + Rate: DefaultRate, + }) +} + +// WithWatcher adds a watcher to the Loader +func (f *Loader) WithWatcher() *Loader { + var filePaths = make([]string, len(f.cfg.Files)) + for i, fi := range f.cfg.Files { + filePaths[i] = fi.Path + } + var fw = kwfile.New( + &kwfile.Config{ + Files: filePaths, + Rate: f.cfg.Rate, + Debug: f.cfg.Debug, + Logger: f.cfg.Logger, + }, + ) + f.FileWatcher = fw + + return f +} + +// Name returns the name of the loader +func (f *Loader) Name() string { return f.cfg.Name } + +// MaxRetry implements konfig.Loader interface and returns the maximum number +// of time Load method can be retried +func (f *Loader) MaxRetry() int { + return f.cfg.MaxRetry +} + +// RetryDelay implements konfig.Loader interface and returns the delay between each retry +func (f *Loader) RetryDelay() time.Duration { + return f.cfg.RetryDelay +} + +// Load implements the konfig.Loader interface. It reads from the file and adds the data to the konfig.Store. +func (f *Loader) Load(cfg konfig.Values) error { + for _, file := range f.cfg.Files { + var fd, err = f.fs.Open(file.Path) + if err != nil { + return err + } + defer fd.Close() + + // we parse the file + if err := file.Parser.Parse(fd, cfg); err != nil { + return err + } + } + return nil +} + +func defaultLogger() nlogger.Logger { + return nlogger.New(os.Stdout, "FILWATCHER | ") +} diff --git a/loader/klfile/fileloader_test.go b/loader/klfile/fileloader_test.go new file mode 100644 index 00000000..cb486e95 --- /dev/null +++ b/loader/klfile/fileloader_test.go @@ -0,0 +1,208 @@ +package klfile + +import ( + "errors" + "io/ioutil" + "strings" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/lalamove/konfig" + "github.com/lalamove/konfig/mocks" + "github.com/lalamove/konfig/parser/kpjson" + "github.com/lalamove/nui/nfs" + "github.com/stretchr/testify/require" +) + +func TestFileLoader(t *testing.T) { + var testCases = []struct { + name string + fileName string + setUp func(ctrl *gomock.Controller, fl *Loader) + err bool + }{ + { + name: "BasicNoErrorLoadOnce", + fileName: "./test", + setUp: func(ctrl *gomock.Controller, fl *Loader) { + var fs = nfs.NewMockFileSystem(ctrl) + var r = ioutil.NopCloser(strings.NewReader( + "FOO=BAR\nBAR=FOO", + )) + fs.EXPECT().Open("./test").Return(r, nil) + fl.fs = fs + fl.cfg.Files[0].Parser.(*mocks.MockParser).EXPECT().Parse(r, konfig.Values{}).Return(nil) + }, + }, + { + name: "ErrorOnFile", + fileName: "./test", + setUp: func(ctrl *gomock.Controller, fl *Loader) { + var fs = nfs.NewMockFileSystem(ctrl) + fs.EXPECT().Open("./test").Return(nil, errors.New("")) + fl.fs = fs + }, + err: true, + }, + { + name: "ErrorInvalidFormat", + fileName: "./test", + setUp: func(ctrl *gomock.Controller, fl *Loader) { + var fs = nfs.NewMockFileSystem(ctrl) + var r = ioutil.NopCloser( + strings.NewReader(`{"test":"test"`), + ) + fs.EXPECT().Open("./test").Return( + r, + nil, + ) + fl.fs = fs + fl.cfg.Files[0].Parser.(*mocks.MockParser). + EXPECT(). + Parse(r, konfig.Values{}). + Return(errors.New("")) + }, + err: true, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + konfig.Init(&konfig.Config{}) + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + var v = konfig.Values{} + + var fl = New(&Config{ + Files: []File{ + { + Path: testCase.fileName, + Parser: mocks.NewMockParser(ctrl), + }, + }, + }) + + testCase.setUp(ctrl, fl) + var err = fl.Load(v) + if testCase.err { + require.NotNil(t, err, "err should not be nil") + return + } + require.Nil(t, err, "err should be nil") + }) + } + +} + +func TestMaxRetryRetryDelay(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + var fl = New(&Config{ + MaxRetry: 10, + RetryDelay: 1 * time.Second, + Files: []File{ + { + Path: "dummy", + Parser: mocks.NewMockParser(ctrl), + }, + }, + }) + require.Equal(t, 10, fl.MaxRetry()) + require.Equal(t, 1*time.Second, fl.RetryDelay()) +} + +func TestNewLoader(t *testing.T) { + t.Run( + "No parser panics", + func(t *testing.T) { + require.Panics(t, func() { + New(&Config{ + Files: []File{ + { + Path: "dummy", + Parser: nil, + }, + }, + }) + }) + }, + ) + + t.Run( + "No files panics", + func(t *testing.T) { + require.Panics(t, func() { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + New(&Config{ + Files: []File{}, + }) + }) + }, + ) + + t.Run( + "With watcher", + func(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + var wl = New(&Config{ + Files: []File{ + { + Path: "fileloader_test.go", + Parser: mocks.NewMockParser(ctrl), + }, + }, + Watch: true, + }) + require.NotNil(t, wl.FileWatcher) + }, + ) +} + +func TestNewFileLoader(t *testing.T) { + t.Run( + "new file loader without watcher", + func(t *testing.T) { + var fl = NewFileLoader("config-files", kpjson.Parser, "foo.json", "bar.json") + + require.Equal( + t, + "foo.json", + fl.cfg.Files[0].Path, + ) + + require.Equal( + t, + "bar.json", + fl.cfg.Files[1].Path, + ) + }, + ) + + t.Run( + "new file loader with watcher", + func(t *testing.T) { + var fl = NewFileLoader("config-files", kpjson.Parser, "./fileloader.go", "./fileloader_test.go").WithWatcher() + + require.Equal( + t, + "./fileloader.go", + fl.cfg.Files[0].Path, + ) + + require.Equal( + t, + "./fileloader_test.go", + fl.cfg.Files[1].Path, + ) + + require.NotNil( + t, + fl.FileWatcher, + ) + }, + ) +} diff --git a/loader/klflag/README.md b/loader/klflag/README.md new file mode 100644 index 00000000..56ab8db9 --- /dev/null +++ b/loader/klflag/README.md @@ -0,0 +1,16 @@ +# Flag Loader +Loads config values from command line flags + +# Usage + +Basic usage with command line FlagSet +```go +flagLoader := klflag.New(&klflag.Config{}) +``` + +With a nstrings.Replacer for keys +```go +flagLoader := klflag.New(&klflag.Config{ + Replacer: strings.NewReplacer(".", "-") +}) +``` diff --git a/loader/klflag/flagloader.go b/loader/klflag/flagloader.go new file mode 100644 index 00000000..0d2395fa --- /dev/null +++ b/loader/klflag/flagloader.go @@ -0,0 +1,77 @@ +package klflag + +import ( + "flag" + "time" + + "github.com/lalamove/konfig" + "github.com/lalamove/nui/nstrings" +) + +var _ konfig.Loader = (*Loader)(nil) + +const defaultName = "flag" + +// Config is the config for the Flag Loader +type Config struct { + // Name is the name of the loader + Name string + // FlagSet is the flag set from which to load flags in config + // default value is flag.CommandLine + FlagSet *flag.FlagSet + // Prefix is the prefix to append before each flag to be added in the konfig.Store + Prefix string + // Replacer is a replacer to apply on flags to be added in the konfig.Store + Replacer nstrings.Replacer + // MaxRetry is the maximum number of times to retry + MaxRetry int + // RetryDelay is the delay between each retry + RetryDelay time.Duration +} + +// Loader is a loader for command line flags +type Loader struct { + cfg *Config +} + +// New creates a new Loader with the given Config cfg +func New(cfg *Config) *Loader { + if cfg.FlagSet == nil { + cfg.FlagSet = flag.CommandLine + } + + if cfg.Name == "" { + cfg.Name = defaultName + } + + return &Loader{ + cfg: cfg, + } +} + +// Name returns the name of the loader +func (l *Loader) Name() string { return l.cfg.Name } + +// Load implements konfig.Loader interface, it loads flags from the FlagSet given in config +// into the konfig.Store +func (l *Loader) Load(s konfig.Values) error { + l.cfg.FlagSet.VisitAll(func(f *flag.Flag) { + var n = f.Name + if l.cfg.Replacer != nil { + n = l.cfg.Replacer.Replace(n) + } + s.Set(l.cfg.Prefix+n, f.Value.String()) + }) + return nil +} + +// MaxRetry implements the konfig.Loader interface, it returns the max number of times a Load can be retried +// if it fails +func (l *Loader) MaxRetry() int { + return l.cfg.MaxRetry +} + +// RetryDelay implements the konfig.Loader interface, is the delay between each retry +func (l *Loader) RetryDelay() time.Duration { + return l.cfg.RetryDelay +} diff --git a/loader/klflag/flagloader_test.go b/loader/klflag/flagloader_test.go new file mode 100644 index 00000000..a342a08b --- /dev/null +++ b/loader/klflag/flagloader_test.go @@ -0,0 +1,70 @@ +package klflag + +import ( + "flag" + "strings" + "testing" + "time" + + "github.com/lalamove/konfig" + "github.com/stretchr/testify/require" +) + +func TestFlagLoader(t *testing.T) { + t.Run( + "multiple flags", + func(t *testing.T) { + konfig.Init(konfig.DefaultConfig()) + + var f = flag.NewFlagSet("foo", flag.ContinueOnError) + f.Bool("foo", true, "") + + var loader = New(&Config{ + FlagSet: f, + }) + + var v = konfig.Values{} + + loader.Load(v) + require.Equal(t, "true", v["foo"]) + }, + ) + + t.Run( + "with replacer and prefix", + func(t *testing.T) { + konfig.Init(konfig.DefaultConfig()) + + var fs = flag.NewFlagSet("foo", flag.ContinueOnError) + fs.Bool("foo", true, "usage") + + var loader = New(&Config{ + Prefix: "foo_", + Replacer: strings.NewReplacer("foo", "bar"), + FlagSet: fs, + }) + + var v = konfig.Values{} + + loader.Load(v) + require.Equal(t, "true", v["foo_bar"]) + }, + ) + + t.Run( + "default flag set", + func(t *testing.T) { + var loader = New(&Config{}) + require.True(t, loader.cfg.FlagSet == flag.CommandLine) + }, + ) + + t.Run( + "max retry retry delay", + func(t *testing.T) { + var loader = New(&Config{MaxRetry: 1, RetryDelay: 10 * time.Second}) + require.Equal(t, 1, loader.MaxRetry()) + require.Equal(t, 10*time.Second, loader.RetryDelay()) + }, + ) +} diff --git a/loader/klhttp/README.md b/loader/klhttp/README.md new file mode 100644 index 00000000..e8b40c86 --- /dev/null +++ b/loader/klhttp/README.md @@ -0,0 +1,19 @@ +# HTTP Loader +Loads config from a source over HTTP + +# Usage + +Basic usage with a json source and a poll watcher +```go +httpLoader := klhttp.New(&klhttp.Config{ + Sources: []Source{ + { + URL: "https://konfig.io/config.json", + Method: "GET", + Parser: kpjson.Parser, + }, + }, + Watch: true, + Rater: kwpoll.Time(10 * time.Second), // Rater is the rater for the poll watcher +}) +``` diff --git a/loader/klhttp/httploader.go b/loader/klhttp/httploader.go new file mode 100644 index 00000000..86e49f0d --- /dev/null +++ b/loader/klhttp/httploader.go @@ -0,0 +1,154 @@ +package klhttp + +import ( + "errors" + "io" + "io/ioutil" + "net/http" + "time" + + "github.com/lalamove/konfig" + "github.com/lalamove/konfig/parser" + "github.com/lalamove/konfig/watcher/kwpoll" +) + +var ( + defaultRate = 10 * time.Second + // ErrNoSources is the error thrown when creating an Loader without sources + ErrNoSources = errors.New("No sources provided") +) + +const defaultName = "http" + +// Client is the interface used to send the HTTP request. +// It is implemented by http.Client. +type Client interface { + Do(*http.Request) (*http.Response, error) +} + +// Source is an HTTP source and a Parser +type Source struct { + URL string + Method string + Body io.Reader + Parser parser.Parser + // Prepare is a function to modify request before sending it + Prepare func(*http.Request) + // StatusCode is the status code expected from this source + // If the status code of the response is different, an error is returned. + // Default is 200. + StatusCode int +} + +// Config is the configuration of the Loader +type Config struct { + // Name is the name of the loader + Name string + // Sources is a list of remote sources + Sources []Source + // Client is the client used to fetch the file, default is http.DefaultClient + Client Client + // MaxRetry is the maximum number of retries when an error occurs + MaxRetry int + // RetryDelay is the delay between each retry + RetryDelay time.Duration + // Watch sets the wether changes should be watched + Watch bool + // Rater is the rater to pass to the poll write + Rater kwpoll.Rater +} + +// Loader loads a configuration remotely +type Loader struct { + *kwpoll.PollWatcher + cfg *Config +} + +// New returns a new Loader with the given Config. +func New(cfg *Config) *Loader { + if cfg.Client == nil { + cfg.Client = http.DefaultClient + } + + if cfg.Sources == nil || len(cfg.Sources) == 0 { + panic(ErrNoSources) + } + + if cfg.Name == "" { + cfg.Name = defaultName + } + + var l = &Loader{ + cfg: cfg, + } + + for i, source := range cfg.Sources { + if source.Method == "" { + source.Method = http.MethodGet + } + cfg.Sources[i] = source + } + + if cfg.Watch { + var r, err = l.Get() + if err != nil { + panic(err) + } + l.PollWatcher = kwpoll.New(&kwpoll.Config{ + Getter: l, + Rater: cfg.Rater, + InitValue: r, + }) + } + + return l +} + +// Name returns the name of the loader +func (r *Loader) Name() string { return r.cfg.Name } + +// Load loads the config from sources and parses the response +func (r *Loader) Load(s konfig.Values) error { + for _, source := range r.cfg.Sources { + if b, err := source.Do(r.cfg.Client); err == nil { + if err := source.Parser.Parse(b, s); err != nil { + return err + } + } else { + return err + } + } + return nil +} + +// Get implements the kwpoll.Getter interface. +// It calls all sources and combines them in a slice an returns it. +func (r *Loader) Get() (interface{}, error) { + var result = make([][]byte, len(r.cfg.Sources)) + + for i, source := range r.cfg.Sources { + + if b, err := source.Do(r.cfg.Client); err == nil { + + var b, err = ioutil.ReadAll(b) + if err != nil { + return nil, err + } + result[i] = b + + } else { + return nil, err + } + } + return result, nil +} + +// MaxRetry returns the MaxRetry config property, it implements the konfig.Loader interface +func (r *Loader) MaxRetry() int { + return r.cfg.MaxRetry +} + +// RetryDelay returns the RetryDelay config property, it implements the konfig.Loader interface +func (r *Loader) RetryDelay() time.Duration { + return r.cfg.RetryDelay +} diff --git a/loader/klhttp/httploader_test.go b/loader/klhttp/httploader_test.go new file mode 100644 index 00000000..472f8c06 --- /dev/null +++ b/loader/klhttp/httploader_test.go @@ -0,0 +1,386 @@ +package klhttp + +import ( + "errors" + "io/ioutil" + "net/http" + "strings" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/lalamove/konfig" + "github.com/lalamove/konfig/mocks" + "github.com/lalamove/konfig/watcher/kwpoll" + "github.com/stretchr/testify/require" +) + +type RequestMatcher struct { + req *http.Request + msg string +} + +func (r *RequestMatcher) Matches(x interface{}) bool { + if v, ok := x.(*http.Request); ok { + if v.Method != r.req.Method || v.URL.String() != r.req.URL.String() { + r.msg = "method or url are different" + return false + } + + if r.req.Body != nil && v.Body == nil { + r.msg = "body are different" + return false + } + + if r.req.Body != nil { + var b, _ = ioutil.ReadAll(r.req.Body) + var b2, _ = ioutil.ReadAll(v.Body) + + if string(b) != string(b2) { + r.msg = "body are different" + return false + } + } + + return true + } + return false +} + +func (r *RequestMatcher) String() string { + return r.msg +} + +func TestLoad(t *testing.T) { + var testCases = []struct { + name string + setUp func(ctrl *gomock.Controller) *Loader + err bool + }{ + { + name: "single source no error get request", + setUp: func(ctrl *gomock.Controller) *Loader { + var c = mocks.NewMockClient(ctrl) + var p1 = mocks.NewMockParser(ctrl) + + var hl = New(&Config{ + Client: c, + Sources: []Source{ + { + URL: "http://source.com", + Parser: p1, + }, + }, + }) + + var r = ioutil.NopCloser(strings.NewReader(``)) + var req, _ = http.NewRequest("GET", "http://source.com", nil) + c.EXPECT().Do(&RequestMatcher{ + req: req, + }).Times(1).Return( + &http.Response{ + StatusCode: 200, + Body: r, + }, + nil, + ) + + p1.EXPECT().Parse(r, konfig.Values{}).Times(1).Return(nil) + + return hl + }, + err: false, + }, + { + name: "multiple sources no error get request", + setUp: func(ctrl *gomock.Controller) *Loader { + var c = mocks.NewMockClient(ctrl) + var p1 = mocks.NewMockParser(ctrl) + var p2 = mocks.NewMockParser(ctrl) + + var hl = New(&Config{ + Client: c, + Sources: []Source{ + { + URL: "http://source.com", + Parser: p1, + }, + { + Method: http.MethodPost, + URL: "http://source.com", + Parser: p2, + }, + }, + }) + + var r = ioutil.NopCloser(strings.NewReader(``)) + var req1, _ = http.NewRequest("GET", "http://source.com", nil) + var req2, _ = http.NewRequest("POST", "http://source.com", nil) + + gomock.InOrder( + c.EXPECT().Do(&RequestMatcher{ + req: req1, + }).Times(1).Return( + &http.Response{ + StatusCode: 200, + Body: r, + }, + nil, + ), + c.EXPECT().Do(&RequestMatcher{ + req: req2, + }).Times(1).Return( + &http.Response{ + StatusCode: 200, + Body: r, + }, + nil, + ), + ) + + p1.EXPECT().Parse(r, konfig.Values{}).Times(1).Return(nil) + p2.EXPECT().Parse(r, konfig.Values{}).Times(1).Return(nil) + + return hl + }, + err: false, + }, + { + name: "multiple sources watch no error get request", + setUp: func(ctrl *gomock.Controller) *Loader { + var c = mocks.NewMockClient(ctrl) + var p1 = mocks.NewMockParser(ctrl) + var p2 = mocks.NewMockParser(ctrl) + + var r = ioutil.NopCloser(strings.NewReader(``)) + var req1, _ = http.NewRequest("GET", "http://source.com", nil) + var req2, _ = http.NewRequest("POST", "http://source.com", nil) + + gomock.InOrder( + c.EXPECT().Do(&RequestMatcher{ + req: req1, + }).Times(1).Return( + &http.Response{ + StatusCode: 200, + Body: r, + }, + nil, + ), + c.EXPECT().Do(&RequestMatcher{ + req: req2, + }).Times(1).Return( + &http.Response{ + StatusCode: 200, + Body: r, + }, + nil, + ), + ) + + var hl = New(&Config{ + Client: c, + Watch: true, + Rater: kwpoll.Time(100 * time.Millisecond), + Sources: []Source{ + { + URL: "http://source.com", + Parser: p1, + }, + { + Method: http.MethodPost, + URL: "http://source.com", + Parser: p2, + }, + }, + }) + + gomock.InOrder( + c.EXPECT().Do(&RequestMatcher{ + req: req1, + }).Times(1).Return( + &http.Response{ + StatusCode: 200, + Body: r, + }, + nil, + ), + c.EXPECT().Do(&RequestMatcher{ + req: req2, + }).Times(1).Return( + &http.Response{ + StatusCode: 200, + Body: r, + }, + nil, + ), + ) + + p1.EXPECT().Parse(r, konfig.Values{}).Times(1).Return(nil) + p2.EXPECT().Parse(r, konfig.Values{}).Times(1).Return(nil) + + return hl + }, + err: false, + }, + { + name: "multiple sources no error get request", + setUp: func(ctrl *gomock.Controller) *Loader { + var c = mocks.NewMockClient(ctrl) + var p1 = mocks.NewMockParser(ctrl) + var p2 = mocks.NewMockParser(ctrl) + + var hl = New(&Config{ + Client: c, + Sources: []Source{ + { + URL: "http://source.com", + Parser: p1, + }, + { + Method: http.MethodPost, + URL: "http://source.com", + Parser: p2, + }, + }, + }) + + var r = ioutil.NopCloser(strings.NewReader(``)) + var req1, _ = http.NewRequest("GET", "http://source.com", nil) + var req2, _ = http.NewRequest("POST", "http://source.com", nil) + + gomock.InOrder( + c.EXPECT().Do(&RequestMatcher{ + req: req1, + }).Times(1).Return( + &http.Response{ + StatusCode: 200, + Body: r, + }, + nil, + ), + c.EXPECT().Do(&RequestMatcher{ + req: req2, + }).Times(1).Return( + nil, + errors.New(""), + ), + ) + + p1.EXPECT().Parse(r, konfig.Values{}).Times(1).Return(nil) + + return hl + }, + err: true, + }, + { + name: "single source error wrong status code", + setUp: func(ctrl *gomock.Controller) *Loader { + var c = mocks.NewMockClient(ctrl) + var p1 = mocks.NewMockParser(ctrl) + + var hl = New(&Config{ + Client: c, + Sources: []Source{ + { + URL: "http://source.com", + Parser: p1, + StatusCode: 201, + }, + }, + }) + + var r = ioutil.NopCloser(strings.NewReader(``)) + var req, _ = http.NewRequest("GET", "http://source.com", nil) + c.EXPECT().Do(&RequestMatcher{ + req: req, + }).Times(1).Return( + &http.Response{ + StatusCode: 400, + Body: r, + }, + nil, + ) + return hl + }, + err: true, + }, + } + + for _, testCase := range testCases { + t.Run( + testCase.name, + func(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + konfig.Init(konfig.DefaultConfig()) + var hl = testCase.setUp(ctrl) + + var err = hl.Load(konfig.Values{}) + if testCase.err { + require.NotNil(t, err, "err should not be nil") + return + } + require.Nil(t, err, "err should be nil") + }, + ) + } +} + +func TestNew(t *testing.T) { + t.Run( + "default http client", + func(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + var p = mocks.NewMockParser(ctrl) + + var hl = New(&Config{ + Sources: []Source{ + { + URL: "http://url.com", + Parser: p, + }, + }, + }) + + require.Equal(t, http.DefaultClient, hl.cfg.Client) + }, + ) + t.Run( + "panic no sources", + func(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + require.Panics(t, func() { + New(&Config{ + Sources: []Source{}, + }) + }) + }, + ) +} + +func TestMaxRetryRetryDelay(t *testing.T) { + + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + var p = mocks.NewMockParser(ctrl) + + var hl = New(&Config{ + MaxRetry: 1, + RetryDelay: 1 * time.Second, + Sources: []Source{ + { + URL: "http://url.com", + Parser: p, + }, + }, + }) + + require.Equal(t, 1*time.Second, hl.RetryDelay()) + require.Equal(t, 1, hl.MaxRetry()) +} diff --git a/loader/klhttp/source.go b/loader/klhttp/source.go new file mode 100644 index 00000000..7680d2a9 --- /dev/null +++ b/loader/klhttp/source.go @@ -0,0 +1,44 @@ +package klhttp + +import ( + "fmt" + "io" + "net/http" +) + +// Do makes an http request and sends the body to the parser +func (s Source) Do(c Client) (io.Reader, error) { + var req, err = http.NewRequest( + s.Method, + s.URL, + s.Body, + ) + if err != nil { + return nil, err + } + + // call the prepare method if there is one + if s.Prepare != nil { + s.Prepare(req) + } + + // make the request + var res *http.Response + res, err = c.Do(req) + if err != nil { + return nil, err + } + + // check status code + if (s.StatusCode != 0 && res.StatusCode != s.StatusCode) || + (res.StatusCode != http.StatusOK) { + + return nil, fmt.Errorf( + "Error while fetching config at %s, status code: %d", + s.URL, + res.StatusCode, + ) + } + + return res.Body, nil +} diff --git a/loader/klvault/README.md b/loader/klvault/README.md new file mode 100644 index 00000000..ed559d33 --- /dev/null +++ b/loader/klvault/README.md @@ -0,0 +1,21 @@ +# Vault Loader +Loads config values from a vault secrets + +# Usage + +Basic usage with Kubernetes auth provider and renewal +```go +vaultLoader := klvault.New(&klvault.Config{ + Secrets: []klvault.Secret{ + { + Key: "/database/creds/db" + }, + }, + Client: vaultClient, // from github.com/hashicorp/vault/api + AuthProvider: k8s.New(&k8s.Config{ + Client: vaultClient, + K8sTokenPath: "/var/run/secrets/kubernetes.io/serviceaccount/token", + }), + Renew: true, +}) +``` diff --git a/loader/klvault/auth/k8s/k8s.go b/loader/klvault/auth/k8s/k8s.go new file mode 100644 index 00000000..fd216834 --- /dev/null +++ b/loader/klvault/auth/k8s/k8s.go @@ -0,0 +1,171 @@ +package k8s + +import ( + "bytes" + "encoding/base64" + "errors" + "io" + "io/ioutil" + "strings" + "time" + + "github.com/lalamove/konfig/loader/klvault" + "github.com/lalamove/nui/nfs" + "github.com/francoispqt/gojay" + vault "github.com/hashicorp/vault/api" +) + +var _ klvault.AuthProvider = (*VaultAuth)(nil) + +const ( + loginPath = "/auth/kubernetes/login" + k8sTokenKeyNamespace = "kubernetes.io/serviceaccount/namespace" + k8sTokenKeyServiceAccount = "kubernetes.io/serviceaccount/service-account.name" +) + +var ( + errNoAuth = errors.New("No authentication in login response") + errNoClient = errors.New("No client provided") + errMalformedToken = errors.New("K8s token is malformed") + fileSystem = nfs.OSFileSystem{} +) + +// VaultAuth is the structure representing a vault authentication provider +type VaultAuth struct { + cfg *Config + k8sToken string + role string + logicalClient klvault.LogicalClient +} + +// Config is the config of a VaultAuth provider +type Config struct { + // Client is the vault client + Client *vault.Client + // K8sTokenPath is the path to the kubernetes service account jwt + K8sTokenPath string + // Role is the role string + Role string + // RoleFunc is a function to build the role + RoleFunc func(string) (string, error) + // FileSystem is the file system to use + // If no value provided it uses the os file system + FileSystem nfs.FileSystem +} + +// New creates a new K8sVaultauth with the given config cfg. +func New(cfg *Config) *VaultAuth { + // if no vault client + if cfg.Client == nil { + panic(errNoClient) + } + // if no file system use the default file system, + if cfg.FileSystem == nil { + cfg.FileSystem = fileSystem + } + + var k8sVault = &VaultAuth{ + cfg: cfg, + logicalClient: cfg.Client.Logical(), + } + + // load the k8s token + var token string + var err error + if token, err = k8sVault.readK8sToken(); err != nil { + panic(err) + } + k8sVault.k8sToken = token + + var role string + // if role is in config, use it + if k8sVault.cfg.Role != "" { + role = k8sVault.cfg.Role + } else if k8sVault.cfg.RoleFunc != nil { + // if we have a role func run it + if role, err = k8sVault.cfg.RoleFunc(token); err != nil { + panic(err) + } + } else { + // use the default role func + if role, err = k8sVault.buildRole(token); err != nil { + panic(err) + } + } + k8sVault.role = role + + return k8sVault +} + +// Token returns a vault token or an error if it encountered one. +// {"jwt": "'"$KUBE_TOKEN"'", "role": "{{ SERVICE_ACCOUNT_NAME }}"} +func (k *VaultAuth) Token() (string, time.Duration, error) { + var s, err = k.logicalClient.Write( + loginPath, + map[string]interface{}{ + "jwt": k.k8sToken, + "role": k.role, + }, + ) + if err != nil { + return "", 0, err + } + // if we don't have auth return an error + if s.Auth == nil { + return "", 0, errNoAuth + } + // return the client token + return s.Auth.ClientToken, time.Duration(s.Auth.LeaseDuration) * time.Second, nil +} + +func (k *VaultAuth) readK8sToken() (string, error) { + var f io.ReadCloser + var err error + if f, err = k.cfg.FileSystem.Open(k.cfg.K8sTokenPath); err != nil { + return "", err + } + + var b []byte + b, err = ioutil.ReadAll(f) + if err != nil { + return "", err + } + return string(b), nil + +} + +func (k *VaultAuth) buildRole(k8sToken string) (string, error) { + // the token is a JWT, we split it by dots and take what's at index 1 + var tokenSpl = strings.Split(k8sToken, ".") + if len(tokenSpl) != 3 { + return "", errMalformedToken + } + + var b64TokenData = tokenSpl[1] + + var tokenData, err = base64.RawStdEncoding.DecodeString(b64TokenData) + if err != nil { + return "", err + } + + var dec = gojay.BorrowDecoder(bytes.NewReader(tokenData)) + defer dec.Release() + + var namespace string + var role string + + err = dec.Decode(gojay.DecodeObjectFunc(func(dec *gojay.Decoder, k string) error { + switch k { + case k8sTokenKeyNamespace: + return dec.String(&namespace) + case k8sTokenKeyServiceAccount: + return dec.String(&role) + } + return nil + })) + + if err != nil { + return "", err + } + return namespace + "-" + role, nil +} diff --git a/loader/klvault/auth/k8s/k8s_test.go b/loader/klvault/auth/k8s/k8s_test.go new file mode 100644 index 00000000..b64af43a --- /dev/null +++ b/loader/klvault/auth/k8s/k8s_test.go @@ -0,0 +1,235 @@ +package k8s + +import ( + "encoding/base64" + "errors" + "io/ioutil" + "strings" + "testing" + "time" + + "github.com/lalamove/konfig/mocks" + "github.com/lalamove/nui/nfs" + "github.com/golang/mock/gomock" + vault "github.com/hashicorp/vault/api" + "github.com/stretchr/testify/require" +) + +func TestNewK8sAuth(t *testing.T) { + t.Run( + "new no error, uses default role builder", + func(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + var fs = nfs.NewMockFileSystem(ctrl) + + fs.EXPECT(). + Open("test"). + Return( + ioutil.NopCloser(strings.NewReader( + "12345."+base64.RawStdEncoding.EncodeToString([]byte( + `{"kubernetes.io/serviceaccount/namespace":"dev","kubernetes.io/serviceaccount/service-account.name":"vault-config-loader"}`, + ))+".12345")), + nil, + ) + + var c, _ = vault.NewClient(vault.DefaultConfig()) + + var k8sAuth = New(&Config{ + K8sTokenPath: "test", + Client: c, + FileSystem: fs, + }) + + require.Equal(t, "dev-vault-config-loader", k8sAuth.role) + }, + ) + + t.Run( + "new no error, uses config role", + func(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + var fs = nfs.NewMockFileSystem(ctrl) + + fs.EXPECT(). + Open("test"). + Return( + ioutil.NopCloser(strings.NewReader( + "12345."+base64.RawStdEncoding.EncodeToString([]byte( + `{"kubernetes.io/serviceaccount/namespace":"dev","kubernetes.io/serviceaccount/service-account.name":"vault-config-loader"}`, + ))+".12345")), + nil, + ) + + var c, _ = vault.NewClient(vault.DefaultConfig()) + + var k8sAuth = New(&Config{ + K8sTokenPath: "test", + Client: c, + FileSystem: fs, + Role: "foobar", + }) + + require.Equal(t, "foobar", k8sAuth.role) + }, + ) + + t.Run( + "new no error, uses config role func", + func(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + var fs = nfs.NewMockFileSystem(ctrl) + + fs.EXPECT(). + Open("test"). + Return( + ioutil.NopCloser(strings.NewReader( + "12345."+base64.RawStdEncoding.EncodeToString([]byte( + `{"kubernetes.io/serviceaccount/namespace":"dev","kubernetes.io/serviceaccount/service-account.name":"vault-config-loader"}`, + ))+".12345")), + nil, + ) + + var c, _ = vault.NewClient(vault.DefaultConfig()) + + var k8sAuth = New(&Config{ + K8sTokenPath: "test", + Client: c, + FileSystem: fs, + RoleFunc: func(string) (string, error) { + return "foobar", nil + }, + }) + + require.Equal(t, "foobar", k8sAuth.role) + }, + ) + + t.Run( + "new panics no client", + func(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + require.Panics(t, func() { + New(&Config{ + K8sTokenPath: "test", + RoleFunc: func(string) (string, error) { + return "foobar", nil + }, + }) + }) + }, + ) +} + +func TestBuildRole(t *testing.T) { + var testCases = []struct { + name string + token string + expectedRole string + err bool + }{ + { + token: "12345." + base64.RawStdEncoding.EncodeToString([]byte( + `{"kubernetes.io/serviceaccount/namespace":"dev","kubernetes.io/serviceaccount/service-account.name":"vault-config-loader"}`, + )) + ".ABCD", + expectedRole: "dev-vault-config-loader", + }, + { + token: "ABCDE", + err: true, + }, + { + token: "12345." + base64.RawStdEncoding.EncodeToString([]byte( + `{"kubernetes.io/serviceaccount/namespace":"dev","kubernetes.io/serviceaccount/service-account.name":"vault-config-loader"}`, + )) + ".ABCD", + expectedRole: "dev-vault-config-loader", + }, + } + + for _, testCase := range testCases { + t.Run( + testCase.name, + func(t *testing.T) { + var k8sAuth = &VaultAuth{} + var s, err = k8sAuth.buildRole(testCase.token) + if testCase.err { + require.NotNil(t, err, "err should not be nil") + return + } + require.Nil(t, err, "err should be nil") + require.Equal(t, testCase.expectedRole, s) + }, + ) + } +} + +func TestToken(t *testing.T) { + t.Run( + "no error", + func(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + var logicalClient = mocks.NewMockLogicalClient(ctrl) + logicalClient.EXPECT().Write( + loginPath, + map[string]interface{}{ + "jwt": "123", + "role": "role", + }, + ).Times(1).Return(&vault.Secret{ + Auth: &vault.SecretAuth{ + ClientToken: "123", + LeaseDuration: 3600, + }, + }, nil) + + var k = &VaultAuth{ + k8sToken: "123", + role: "role", + logicalClient: logicalClient, + } + + var token, d, err = k.Token() + require.Equal(t, "123", token) + require.Equal(t, 3600*time.Second, d) + require.Nil(t, err) + }, + ) + + t.Run( + "error when calling vault", + func(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + var logicalClient = mocks.NewMockLogicalClient(ctrl) + logicalClient.EXPECT().Write( + loginPath, + map[string]interface{}{ + "jwt": "123", + "role": "role", + }, + ).Times(1).Return( + nil, + errors.New("err"), + ) + + var k = &VaultAuth{ + k8sToken: "123", + role: "role", + logicalClient: logicalClient, + } + + var _, _, err = k.Token() + require.NotNil(t, err) + }, + ) +} diff --git a/loader/klvault/auth/token/token.go b/loader/klvault/auth/token/token.go new file mode 100644 index 00000000..a1bc1c54 --- /dev/null +++ b/loader/klvault/auth/token/token.go @@ -0,0 +1,14 @@ +package token + +import "time" + +// Token auth provider +type Token struct { + T string +} + +// Token returns a vault token or an error if it encountered one. +// {"jwt": "'"$KUBE_TOKEN"'", "role": "{{ SERVICE_ACCOUNT_NAME }}"} +func (k *Token) Token() (string, time.Duration, error) { + return k.T, 10 * time.Second, nil +} diff --git a/loader/klvault/authprovider.go b/loader/klvault/authprovider.go new file mode 100644 index 00000000..95d9af95 --- /dev/null +++ b/loader/klvault/authprovider.go @@ -0,0 +1,8 @@ +package klvault + +import "time" + +// AuthProvider is the interface for a Vault authentication provider +type AuthProvider interface { + Token() (string, time.Duration, error) +} diff --git a/loader/klvault/vaultloader.go b/loader/klvault/vaultloader.go new file mode 100644 index 00000000..efdb3c24 --- /dev/null +++ b/loader/klvault/vaultloader.go @@ -0,0 +1,208 @@ +package klvault + +import ( + "errors" + "fmt" + "os" + "sync" + "time" + + vault "github.com/hashicorp/vault/api" + "github.com/lalamove/konfig" + "github.com/lalamove/konfig/watcher/kwpoll" + "github.com/lalamove/nui/nlogger" + "github.com/lalamove/nui/nstrings" +) + +var _ konfig.Loader = (*Loader)(nil) + +var ( + defaultTTL = 45 * time.Minute + // ErrNoClient is the error thrown when trying to create a Loader without vault.Client + ErrNoClient = errors.New("No vault client provided") + // ErrNoAuthProvider is the error thrown when trying to create a Loader without an AuthProvider + ErrNoAuthProvider = errors.New("No auth provider given") + // ErrNoSecretKey is the error thrown when trying to create a Loader without a SecretKey + ErrNoSecretKey = errors.New("No secret key given") +) + +const defaultName = "vault" + +// LogicalClient is a interface for the vault logical client +type LogicalClient interface { + Read(key string) (*vault.Secret, error) + Write(key string, data map[string]interface{}) (*vault.Secret, error) + ReadWithData(key string, data map[string][]string) (*vault.Secret, error) +} + +// Secret is a secret to load +type Secret struct { + // SecretKey is the URL to fetch the secret from (e.g. /v1/database/creds/mydb) + Key string + // KeysPrefix sets a prefix to be prepended to all keys in the config store + KeysPrefix string + // Replacer transforms vault secret's keys + Replacer nstrings.Replacer +} + +// Config is the config for the Loader +type Config struct { + // Name is the name of the loader + Name string + // Secrets is the list of secrets to load + Secrets []Secret + // AuthProvider is the vault auth provider + AuthProvider AuthProvider + // Client is the vault client for the vault loader + Client *vault.Client + // MaxRetry is the maximum number of times the load method can be retried + MaxRetry int + // RetryDelay is the time between each retry + RetryDelay time.Duration + // Debug enables debug mode + Debug bool + // Logger is the logger used for debug logs + Logger nlogger.Logger + // Renew sets wether the vault loader should renew it self + Renew bool +} + +// Loader is the structure representing a Loader +type Loader struct { + *kwpoll.PollWatcher + cfg *Config + logicalClient LogicalClient + mut *sync.Mutex + ttl time.Duration +} + +// New creates a new Loader with the given config +func New(cfg *Config) *Loader { + if cfg.Secrets == nil || len(cfg.Secrets) == 0 { + panic(ErrNoSecretKey) + } + if cfg.AuthProvider == nil { + panic(ErrNoAuthProvider) + } + if cfg.Client == nil { + panic(ErrNoClient) + } + if cfg.Logger == nil { + cfg.Logger = defaultLogger() + } + if cfg.Name == "" { + cfg.Name = defaultName + } + var vl = &Loader{ + cfg: cfg, + logicalClient: cfg.Client.Logical(), + mut: &sync.Mutex{}, + ttl: defaultTTL, + } + + var pw *kwpoll.PollWatcher + if cfg.Renew { + pw = kwpoll.New( + &kwpoll.Config{ + Debug: cfg.Debug, + Logger: cfg.Logger, + Rater: vl, + }, + ) + } + vl.PollWatcher = pw + + return vl +} + +// Name returns the name of the loader +func (vl *Loader) Name() string { return vl.cfg.Name } + +// MaxRetry is the maximum number of times the load method can be retried +func (vl *Loader) MaxRetry() int { + return vl.cfg.MaxRetry +} + +// RetryDelay is the delay between each retry +func (vl *Loader) RetryDelay() time.Duration { + return vl.cfg.RetryDelay +} + +// Load implements konfig.Loader interface. +// It fetches a token from the auth provider and sets the token in the vault client. +// Then it loads the secret and assigns it values to the konfig.Store. +func (vl *Loader) Load(cs konfig.Values) error { + if vl.cfg.Debug { + vl.cfg.Logger.Debug( + "Loading vault config", + ) + } + // everytime we load we get a new token + // maybe we could improve implementation to use a shorter ticker and check if config if different, if yes, reload it + var token, ttl, err = vl.cfg.AuthProvider.Token() + if err != nil { + vl.cfg.Logger.Error(err.Error()) + + return err + } + // we set the token in the client + vl.cfg.Client.SetToken(token) + + var leaseDuration = int(ttl / time.Second) + for _, secret := range vl.cfg.Secrets { + // we fetch our secret + var s *vault.Secret + s, err = vl.logicalClient.Read(secret.Key) + if err != nil { + return err + } + + if vl.cfg.Debug { + vl.cfg.Logger.Debug( + fmt.Sprintf("Got secret, expiring in: %d", s.LeaseDuration), + ) + } + + // if the current secret lease is smaller than the previous smaller lease + // or there is no previous lease + if s.LeaseDuration != 0 && (leaseDuration == 0 || s.LeaseDuration < leaseDuration) { + leaseDuration = s.LeaseDuration + } + + // we set our data on the config store + for k, v := range s.Data { + var nK = secret.KeysPrefix + k + if secret.Replacer != nil { + nK = secret.Replacer.Replace(nK) + } + cs.Set(nK, v) + } + } + + // reset the ttl for renewal + vl.resetTTL(ttl, time.Duration(leaseDuration)*time.Second) + return nil +} + +// Time returns the TTL of the vault loader +// It is used in the ticker watcher a source. +func (vl *Loader) Time() time.Duration { + return vl.ttl +} + +func (vl *Loader) resetTTL(tokenTTL, secretTTL time.Duration) { + var ttl = tokenTTL + if secretTTL < tokenTTL { + ttl = secretTTL + } + ttl = (ttl * 75) / 100 + vl.mut.Lock() + if ttl != vl.ttl { + vl.ttl = ttl + } + vl.mut.Unlock() +} + +func defaultLogger() nlogger.Logger { + return nlogger.New(os.Stdout, "VAULT CONFIG | ") +} diff --git a/loader/klvault/vaultloader_test.go b/loader/klvault/vaultloader_test.go new file mode 100644 index 00000000..1a440dd7 --- /dev/null +++ b/loader/klvault/vaultloader_test.go @@ -0,0 +1,301 @@ +package klvault + +import ( + "errors" + "sync" + "testing" + "time" + + gomock "github.com/golang/mock/gomock" + vault "github.com/hashicorp/vault/api" + "github.com/lalamove/konfig" + "github.com/lalamove/konfig/mocks" + "github.com/stretchr/testify/require" +) + +func TestVaultLoader(t *testing.T) { + var testCases = []struct { + name string + setUp func(ctrl *gomock.Controller) *Loader + asserts func(t *testing.T, vl *Loader, cfg konfig.Values) + err bool + }{ + { + name: "BasicNoError", + setUp: func(ctrl *gomock.Controller) *Loader { + var aP = mocks.NewMockAuthProvider(ctrl) + aP.EXPECT().Token().Return( + "DUMMYTOKEN", + 1*time.Hour, + nil, + ) + + var c, _ = vault.NewClient(vault.DefaultConfig()) + + var vl = New(&Config{ + Client: c, + Secrets: []Secret{ + {Key: "/dummy/secret/path"}, + {Key: "/dummy/secret/path2"}, + }, + AuthProvider: aP, + }) + + var lC = mocks.NewMockLogicalClient(ctrl) + vl.logicalClient = lC + lC.EXPECT().Read("/dummy/secret/path").Return( + &vault.Secret{ + Data: map[string]interface{}{ + "FOO": "BAR", + }, + LeaseDuration: int(2 * time.Hour / time.Second), + }, + nil, + ) + + lC.EXPECT().Read("/dummy/secret/path2").Return( + &vault.Secret{ + Data: map[string]interface{}{ + "BAR": "FOO", + }, + LeaseDuration: int(1 * time.Hour / time.Second), + }, + nil, + ) + + return vl + }, + asserts: func(t *testing.T, vl *Loader, cfg konfig.Values) { + require.Equal( + t, + vl.ttl, + 45*time.Minute, + ) + require.Equal( + t, + "BAR", + cfg["FOO"], + ) + require.Equal( + t, + "FOO", + cfg["BAR"], + ) + }, + }, + { + name: "ErrorOnAuthProvider", + err: true, + setUp: func(ctrl *gomock.Controller) *Loader { + var aP = mocks.NewMockAuthProvider(ctrl) + aP.EXPECT().Token().Return( + "", + time.Duration(0), + errors.New(""), + ) + + var c, _ = vault.NewClient(vault.DefaultConfig()) + + var vl = New(&Config{ + Client: c, + + Secrets: []Secret{{Key: "/dummy/secret/path"}}, + AuthProvider: aP, + }) + return vl + }, + asserts: func(t *testing.T, vl *Loader, cfg konfig.Values) {}, + }, + { + name: "ErrorFetchingSecret", + err: true, + setUp: func(ctrl *gomock.Controller) *Loader { + var aP = mocks.NewMockAuthProvider(ctrl) + aP.EXPECT().Token().Return( + "DUMMYTOKEN", + 1*time.Hour, + nil, + ) + + var c, _ = vault.NewClient(vault.DefaultConfig()) + + var vl = New(&Config{ + Client: c, + Secrets: []Secret{{Key: "/dummy/secret/path"}}, + AuthProvider: aP, + }) + + var lC = mocks.NewMockLogicalClient(ctrl) + vl.logicalClient = lC + lC.EXPECT().Read("/dummy/secret/path").Return( + nil, + errors.New(""), + ) + + return vl + }, + asserts: func(t *testing.T, vl *Loader, cfg konfig.Values) {}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + var vl = testCase.setUp(ctrl) + konfig.Init(&konfig.Config{}) + var c = konfig.Values{} + + var err = vl.Load(c) + + if testCase.err { + require.NotNil(t, err, "err should not be nil") + return + } + + require.Nil(t, err, "err should be nil") + testCase.asserts(t, vl, c) + }) + } +} + +func TestResetTTL(t *testing.T) { + var testCases = []struct { + name string + tokenTTL time.Duration + secretTTL time.Duration + expectedTTL time.Duration + }{ + { + name: "token TTL is smaller than secret TTL", + tokenTTL: 1 * time.Hour, + secretTTL: 2 * time.Hour, + expectedTTL: 45 * time.Minute, + }, + { + name: "token TTL is smaller than secret TTL", + tokenTTL: 1 * time.Hour, + secretTTL: 30 * time.Minute, + expectedTTL: 1350 * time.Second, + }, + } + + for _, testCase := range testCases { + t.Run( + testCase.name, + func(t *testing.T) { + var vl = &Loader{ + mut: &sync.Mutex{}, + } + vl.resetTTL(testCase.tokenTTL, testCase.secretTTL) + require.Equal(t, testCase.expectedTTL, vl.ttl) + }, + ) + } +} + +func TestNew(t *testing.T) { + t.Run( + "no secret key panics", + func(t *testing.T) { + require.Panics( + t, + func() { + New(&Config{}) + }, + ) + }, + ) + + t.Run( + "no auth provider panics", + func(t *testing.T) { + require.Panics( + t, + func() { + New(&Config{ + Secrets: []Secret{{Key: "/dummy/secret/path"}}, + }) + }, + ) + }, + ) + + t.Run( + "no vault client panics", + func(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + var aP = mocks.NewMockAuthProvider(ctrl) + require.Panics( + t, + func() { + New(&Config{ + Secrets: []Secret{{Key: "/dummy/secret/path"}}, + AuthProvider: aP, + }) + }, + ) + }, + ) + + t.Run( + "no panic, no renewal", + func(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + var aP = mocks.NewMockAuthProvider(ctrl) + var c, _ = vault.NewClient( + vault.DefaultConfig(), + ) + var vl = New(&Config{ + + Secrets: []Secret{{Key: "/dummy/secret/path"}}, + AuthProvider: aP, + Client: c, + }) + + require.Nil(t, vl.PollWatcher) + }, + ) + + t.Run( + "no panic, with renewal", + func(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + var aP = mocks.NewMockAuthProvider(ctrl) + var c, _ = vault.NewClient( + vault.DefaultConfig(), + ) + var vl = New(&Config{ + Secrets: []Secret{{Key: "/dummy/secretr/path"}}, + AuthProvider: aP, + Client: c, + Renew: true, + }) + + require.NotNil(t, vl.PollWatcher) + }, + ) +} + +func TestMaxRetryRetryDelay(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + var aP = mocks.NewMockAuthProvider(ctrl) + var c, _ = vault.NewClient( + vault.DefaultConfig(), + ) + var vl = New(&Config{ + Secrets: []Secret{{Key: "/dummy/secretr/path"}}, + AuthProvider: aP, + Client: c, + Renew: true, + MaxRetry: 1, + RetryDelay: 1 * time.Second, + }) + + require.Equal(t, 1, vl.MaxRetry()) + require.Equal(t, 1*time.Second, vl.RetryDelay()) +} diff --git a/loader_mock_test.go b/loader_mock_test.go new file mode 100644 index 00000000..b95fe5f3 --- /dev/null +++ b/loader_mock_test.go @@ -0,0 +1,90 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./loader.go + +// Package konfig is a generated GoMock package. +package konfig + +import ( + gomock "github.com/golang/mock/gomock" + reflect "reflect" + time "time" +) + +// MockLoader is a mock of Loader interface +type MockLoader struct { + ctrl *gomock.Controller + recorder *MockLoaderMockRecorder +} + +// MockLoaderMockRecorder is the mock recorder for MockLoader +type MockLoaderMockRecorder struct { + mock *MockLoader +} + +// NewMockLoader creates a new mock instance +func NewMockLoader(ctrl *gomock.Controller) *MockLoader { + mock := &MockLoader{ctrl: ctrl} + mock.recorder = &MockLoaderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockLoader) EXPECT() *MockLoaderMockRecorder { + return m.recorder +} + +// Name mocks base method +func (m *MockLoader) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name +func (mr *MockLoaderMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockLoader)(nil).Name)) +} + +// Load mocks base method +func (m *MockLoader) Load(arg0 Values) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Load", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Load indicates an expected call of Load +func (mr *MockLoaderMockRecorder) Load(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Load", reflect.TypeOf((*MockLoader)(nil).Load), arg0) +} + +// MaxRetry mocks base method +func (m *MockLoader) MaxRetry() int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MaxRetry") + ret0, _ := ret[0].(int) + return ret0 +} + +// MaxRetry indicates an expected call of MaxRetry +func (mr *MockLoaderMockRecorder) MaxRetry() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MaxRetry", reflect.TypeOf((*MockLoader)(nil).MaxRetry)) +} + +// RetryDelay mocks base method +func (m *MockLoader) RetryDelay() time.Duration { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RetryDelay") + ret0, _ := ret[0].(time.Duration) + return ret0 +} + +// RetryDelay indicates an expected call of RetryDelay +func (mr *MockLoaderMockRecorder) RetryDelay() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RetryDelay", reflect.TypeOf((*MockLoader)(nil).RetryDelay)) +} diff --git a/loader_test.go b/loader_test.go new file mode 100644 index 00000000..4e332df9 --- /dev/null +++ b/loader_test.go @@ -0,0 +1,280 @@ +package konfig + +import ( + "errors" + "testing" + time "time" + + gomock "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" +) + +func TestLoaderHooksRun(t *testing.T) { + t.Run( + "run all hooks no error", + func(t *testing.T) { + var i int + var loaderHooks = LoaderHooks{ + func(Store) error { + i = i + 1 + return nil + }, + func(Store) error { + i = i + 2 + return nil + }, + func(Store) error { + i = i + 3 + return nil + }, + } + var err = loaderHooks.Run(Instance()) + require.Nil(t, err, "err should be nil") + require.Equal(t, 6, i, "all hooks should have run") + }, + ) + + t.Run( + "run one hook and error", + func(t *testing.T) { + var i int + var loaderHooks = LoaderHooks{ + func(Store) error { + i = i + 1 + return errors.New("err") + }, + func(Store) error { + i = i + 2 + return nil + }, + func(Store) error { + i = i + 3 + return nil + }, + } + var err = loaderHooks.Run(Instance()) + require.NotNil(t, err, "err should not be nil") + require.Equal(t, 1, i, "one hook should have run") + }, + ) +} + +func TestLoaderLoadRetry(t *testing.T) { + var testCases = []struct { + name string + err bool + build func(ctrl *gomock.Controller) *loaderWatcher + }{ + { + name: "success, no loader hooks, no retry", + build: func(ctrl *gomock.Controller) *loaderWatcher { + var mockW = NewMockWatcher(ctrl) + var mockL = NewMockLoader(ctrl) + mockL.EXPECT().Load(Values{}).Return(nil) + + var wl = &loaderWatcher{ + Watcher: mockW, + Loader: mockL, + loaderHooks: nil, + } + return wl + }, + }, + { + name: "success, no loader hooks, 1 retrty", + build: func(ctrl *gomock.Controller) *loaderWatcher { + var mockW = NewMockWatcher(ctrl) + var mockL = NewMockLoader(ctrl) + gomock.InOrder( + mockL.EXPECT().Load(Values{}).Return(errors.New("")), + mockL.EXPECT().RetryDelay().Return(1*time.Millisecond), + mockL.EXPECT().MaxRetry().Return(1), + mockL.EXPECT().Load(Values{}).Return(nil), + ) + var wl = &loaderWatcher{ + Watcher: mockW, + Loader: mockL, + loaderHooks: nil, + } + return wl + }, + }, + { + name: "error, no loader hooks, 1 retry", + err: true, + build: func(ctrl *gomock.Controller) *loaderWatcher { + var mockW = NewMockWatcher(ctrl) + var mockL = NewMockLoader(ctrl) + gomock.InOrder( + mockL.EXPECT().Load(Values{}).Return(errors.New("")), + mockL.EXPECT().RetryDelay().Return(1*time.Millisecond), + mockL.EXPECT().MaxRetry().Return(1), + mockL.EXPECT().Load(Values{}).Return(errors.New("")), + mockL.EXPECT().RetryDelay().Return(1*time.Millisecond), + mockL.EXPECT().MaxRetry().Return(1), + ) + var wl = &loaderWatcher{ + Watcher: mockW, + Loader: mockL, + loaderHooks: nil, + } + return wl + }, + }, + { + name: "success, 2 loader hooks, 1 retry", + build: func(ctrl *gomock.Controller) *loaderWatcher { + var mockW = NewMockWatcher(ctrl) + var mockL = NewMockLoader(ctrl) + gomock.InOrder( + mockL.EXPECT().Load(Values{}).Return(errors.New("")), + mockL.EXPECT().RetryDelay().Return(1*time.Millisecond), + mockL.EXPECT().MaxRetry().Return(1), + mockL.EXPECT().Load(Values{}).Return(nil), + ) + var wl = &loaderWatcher{ + Watcher: mockW, + Loader: mockL, + loaderHooks: LoaderHooks{ + func(Store) error { + return nil + }, + func(Store) error { + return nil + }, + }, + } + return wl + }, + }, + { + name: "error, 2 loader hooks, 1 retry", + err: true, + build: func(ctrl *gomock.Controller) *loaderWatcher { + var mockW = NewMockWatcher(ctrl) + var mockL = NewMockLoader(ctrl) + gomock.InOrder( + mockL.EXPECT().Load(Values{}).Return(errors.New("")), + mockL.EXPECT().RetryDelay().Return(1*time.Millisecond), + mockL.EXPECT().MaxRetry().Return(1), + mockL.EXPECT().Load(Values{}).Return(nil), + ) + var wl = &loaderWatcher{ + Watcher: mockW, + Loader: mockL, + loaderHooks: LoaderHooks{ + func(Store) error { + return nil + }, + func(Store) error { + return errors.New("") + }, + }, + } + return wl + }, + }, + } + + for _, testCase := range testCases { + t.Run( + testCase.name, + func(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + reset() + var c = instance() + c.cfg.NoExitOnError = true + var err = c.loaderLoadRetry(testCase.build(ctrl), 0) + if testCase.err { + require.NotNil(t, err, "err should not be nil") + return + } + require.Nil(t, err, "err should be nil") + }, + ) + } +} + +func TestLoaderLoadWatch(t *testing.T) { + var testCases = []struct { + name string + err bool + build func(ctrl *gomock.Controller) *loaderWatcher + }{ + { + name: "success, no errors", + build: func(ctrl *gomock.Controller) *loaderWatcher { + var mockW = NewMockWatcher(ctrl) + var mockL = NewMockLoader(ctrl) + + mockL.EXPECT().Name().MinTimes(1).Return("test") + mockL.EXPECT().Load(Values{}).Return(nil) + mockW.EXPECT().Start().Return(nil) + mockW.EXPECT().Watch().Return(nil) + mockW.EXPECT().Done().Return(nil) + + var wl = &loaderWatcher{ + Watcher: mockW, + Loader: mockL, + loaderHooks: nil, + } + return wl + }, + }, + { + name: "success, errors load", + err: true, + build: func(ctrl *gomock.Controller) *loaderWatcher { + var mockW = NewMockWatcher(ctrl) + var mockL = NewMockLoader(ctrl) + + mockL.EXPECT().Name().MinTimes(1).Return("test") + mockL.EXPECT().Load(Values{}).Times(4).Return(errors.New("")) + mockL.EXPECT().MaxRetry().Times(4).Return(3) + mockL.EXPECT().RetryDelay().Times(4).Return(50 * time.Millisecond) + mockW.EXPECT().Close().MinTimes(1).Return(nil) + + var wl = &loaderWatcher{ + Watcher: mockW, + Loader: mockL, + loaderHooks: nil, + } + return wl + }, + }, + } + + for _, testCase := range testCases { + t.Run( + testCase.name, + func(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + reset() + + var c = New(&Config{ + Metrics: true, + }) + + c.RegisterLoaderWatcher( + testCase.build(ctrl), + ) + c.(*store).cfg.NoExitOnError = true + + var err = c.LoadWatch() + + if testCase.err { + require.NotNil(t, err, "err should not be nil") + return + } + + require.Nil(t, err, "err should be nil") + + time.Sleep(300 * time.Millisecond) + }, + ) + } +} diff --git a/loaderwatcher.go b/loaderwatcher.go new file mode 100644 index 00000000..359b0bb7 --- /dev/null +++ b/loaderwatcher.go @@ -0,0 +1,40 @@ +package konfig + +// LoaderWatcher is an interface that implements both loader and watcher +type LoaderWatcher interface { + Loader + Watcher +} + +type loaderWatcher struct { + Loader + Watcher + values Values + name string + s *store + metrics *loaderMetrics + loaderHooks LoaderHooks +} + +// NewLoaderWatcher creates a new LoaderWatcher from a Loader and a Watcher +func NewLoaderWatcher(l Loader, w Watcher) LoaderWatcher { + return &loaderWatcher{ + Loader: l, + Watcher: w, + } +} + +func (c *store) newLoaderWatcher(l Loader, w Watcher, loaderHooks LoaderHooks) *loaderWatcher { + var lw = &loaderWatcher{ + Loader: l, + Watcher: w, + s: c, + loaderHooks: loaderHooks, + } + + if c.cfg.Metrics { + lw.setMetrics() + } + + return lw +} diff --git a/metrics.go b/metrics.go new file mode 100644 index 00000000..78e0a6cd --- /dev/null +++ b/metrics.go @@ -0,0 +1,79 @@ +package konfig + +import "github.com/prometheus/client_golang/prometheus" + +var ( + // MetricsConfigReload is the label for the prometheus counter for loader reload + MetricsConfigReload = "konfig_loader_reload" + // MetricsConfigReloadDuration is the label for the prometheus summary vector for loader reload duration + MetricsConfigReloadDuration = "konfig_loader_reload_duration" +) + +const ( + metricsSuccessLabel = "success" + metricsFailureLabel = "failure" +) + +// LoaderMetrics is the structure holding the promtheus metrics objects +type loaderMetrics struct { + configReloadSuccess prometheus.Counter + configReloadFailure prometheus.Counter + configReloadDuration prometheus.Observer +} + +func (lw *loaderWatcher) setMetrics() { + var ( + configReloadCounterVec = lw.s.metrics[MetricsConfigReload].(*prometheus.CounterVec) + configReloadDurationSummaryVec = lw.s.metrics[MetricsConfigReloadDuration].(*prometheus.SummaryVec) + ) + + lw.metrics = &loaderMetrics{ + configReloadSuccess: configReloadCounterVec. + WithLabelValues( + metricsSuccessLabel, + lw.s.cfg.Name, + lw.Name(), + ), + configReloadFailure: configReloadCounterVec. + WithLabelValues( + metricsFailureLabel, + lw.s.cfg.Name, + lw.Name(), + ), + configReloadDuration: configReloadDurationSummaryVec. + WithLabelValues( + lw.s.cfg.Name, + lw.Name(), + ), + } +} + +func (c *store) initMetrics() { + c.metrics = map[string]prometheus.Collector{ + MetricsConfigReload: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: MetricsConfigReload, + Help: "Number of config loader reload", + }, + []string{"result", "store_name", "loader_name"}, + ), + MetricsConfigReloadDuration: prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Name: MetricsConfigReloadDuration, + Help: "Histogram for the config reload duration", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, + []string{"store_name", "loader_name"}, + ), + } +} + +func (c *store) registerMetrics() error { + for _, metric := range c.metrics { + var err = prometheus.Register(metric) + if err != nil && err != err.(prometheus.AlreadyRegisteredError) { + return err + } + } + return nil +} diff --git a/metrics_test.go b/metrics_test.go new file mode 100644 index 00000000..8ca9640e --- /dev/null +++ b/metrics_test.go @@ -0,0 +1 @@ +package konfig diff --git a/mocks/authprovider_mock.go b/mocks/authprovider_mock.go new file mode 100644 index 00000000..fba0ed07 --- /dev/null +++ b/mocks/authprovider_mock.go @@ -0,0 +1,50 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./loader/klvault/authprovider.go + +// Package mocks is a generated GoMock package. +package mocks + +import ( + gomock "github.com/golang/mock/gomock" + reflect "reflect" + time "time" +) + +// MockAuthProvider is a mock of AuthProvider interface +type MockAuthProvider struct { + ctrl *gomock.Controller + recorder *MockAuthProviderMockRecorder +} + +// MockAuthProviderMockRecorder is the mock recorder for MockAuthProvider +type MockAuthProviderMockRecorder struct { + mock *MockAuthProvider +} + +// NewMockAuthProvider creates a new mock instance +func NewMockAuthProvider(ctrl *gomock.Controller) *MockAuthProvider { + mock := &MockAuthProvider{ctrl: ctrl} + mock.recorder = &MockAuthProviderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockAuthProvider) EXPECT() *MockAuthProviderMockRecorder { + return m.recorder +} + +// Token mocks base method +func (m *MockAuthProvider) Token() (string, time.Duration, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Token") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(time.Duration) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// Token indicates an expected call of Token +func (mr *MockAuthProviderMockRecorder) Token() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Token", reflect.TypeOf((*MockAuthProvider)(nil).Token)) +} diff --git a/mocks/client_mock.go b/mocks/client_mock.go new file mode 100644 index 00000000..ace67bef --- /dev/null +++ b/mocks/client_mock.go @@ -0,0 +1,49 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./loader/klhttp/httploader.go + +// Package mocks is a generated GoMock package. +package mocks + +import ( + gomock "github.com/golang/mock/gomock" + http "net/http" + reflect "reflect" +) + +// MockClient is a mock of Client interface +type MockClient struct { + ctrl *gomock.Controller + recorder *MockClientMockRecorder +} + +// MockClientMockRecorder is the mock recorder for MockClient +type MockClientMockRecorder struct { + mock *MockClient +} + +// NewMockClient creates a new mock instance +func NewMockClient(ctrl *gomock.Controller) *MockClient { + mock := &MockClient{ctrl: ctrl} + mock.recorder = &MockClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockClient) EXPECT() *MockClientMockRecorder { + return m.recorder +} + +// Do mocks base method +func (m *MockClient) Do(arg0 *http.Request) (*http.Response, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Do", arg0) + ret0, _ := ret[0].(*http.Response) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Do indicates an expected call of Do +func (mr *MockClientMockRecorder) Do(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Do", reflect.TypeOf((*MockClient)(nil).Do), arg0) +} diff --git a/mocks/contexter_mock.go b/mocks/contexter_mock.go new file mode 100644 index 00000000..70ed2b73 --- /dev/null +++ b/mocks/contexter_mock.go @@ -0,0 +1,80 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/lalamove/nui/ncontext (interfaces: Contexter) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + gomock "github.com/golang/mock/gomock" + reflect "reflect" + time "time" +) + +// MockContexter is a mock of Contexter interface +type MockContexter struct { + ctrl *gomock.Controller + recorder *MockContexterMockRecorder +} + +// MockContexterMockRecorder is the mock recorder for MockContexter +type MockContexterMockRecorder struct { + mock *MockContexter +} + +// NewMockContexter creates a new mock instance +func NewMockContexter(ctrl *gomock.Controller) *MockContexter { + mock := &MockContexter{ctrl: ctrl} + mock.recorder = &MockContexterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockContexter) EXPECT() *MockContexterMockRecorder { + return m.recorder +} + +// WithCancel mocks base method +func (m *MockContexter) WithCancel(arg0 context.Context) (context.Context, context.CancelFunc) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WithCancel", arg0) + ret0, _ := ret[0].(context.Context) + ret1, _ := ret[1].(context.CancelFunc) + return ret0, ret1 +} + +// WithCancel indicates an expected call of WithCancel +func (mr *MockContexterMockRecorder) WithCancel(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WithCancel", reflect.TypeOf((*MockContexter)(nil).WithCancel), arg0) +} + +// WithDeadline mocks base method +func (m *MockContexter) WithDeadline(arg0 context.Context, arg1 time.Time) (context.Context, context.CancelFunc) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WithDeadline", arg0, arg1) + ret0, _ := ret[0].(context.Context) + ret1, _ := ret[1].(context.CancelFunc) + return ret0, ret1 +} + +// WithDeadline indicates an expected call of WithDeadline +func (mr *MockContexterMockRecorder) WithDeadline(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WithDeadline", reflect.TypeOf((*MockContexter)(nil).WithDeadline), arg0, arg1) +} + +// WithTimeout mocks base method +func (m *MockContexter) WithTimeout(arg0 context.Context, arg1 time.Duration) (context.Context, context.CancelFunc) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WithTimeout", arg0, arg1) + ret0, _ := ret[0].(context.Context) + ret1, _ := ret[1].(context.CancelFunc) + return ret0, ret1 +} + +// WithTimeout indicates an expected call of WithTimeout +func (mr *MockContexterMockRecorder) WithTimeout(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WithTimeout", reflect.TypeOf((*MockContexter)(nil).WithTimeout), arg0, arg1) +} diff --git a/mocks/getter_mock.go b/mocks/getter_mock.go new file mode 100644 index 00000000..d98e23a2 --- /dev/null +++ b/mocks/getter_mock.go @@ -0,0 +1,86 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./watcher/kwpoll/pollwatcher.go + +// Package mocks is a generated GoMock package. +package mocks + +import ( + gomock "github.com/golang/mock/gomock" + reflect "reflect" + time "time" +) + +// MockRater is a mock of Rater interface +type MockRater struct { + ctrl *gomock.Controller + recorder *MockRaterMockRecorder +} + +// MockRaterMockRecorder is the mock recorder for MockRater +type MockRaterMockRecorder struct { + mock *MockRater +} + +// NewMockRater creates a new mock instance +func NewMockRater(ctrl *gomock.Controller) *MockRater { + mock := &MockRater{ctrl: ctrl} + mock.recorder = &MockRaterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockRater) EXPECT() *MockRaterMockRecorder { + return m.recorder +} + +// Time mocks base method +func (m *MockRater) Time() time.Duration { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Time") + ret0, _ := ret[0].(time.Duration) + return ret0 +} + +// Time indicates an expected call of Time +func (mr *MockRaterMockRecorder) Time() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Time", reflect.TypeOf((*MockRater)(nil).Time)) +} + +// MockGetter is a mock of Getter interface +type MockGetter struct { + ctrl *gomock.Controller + recorder *MockGetterMockRecorder +} + +// MockGetterMockRecorder is the mock recorder for MockGetter +type MockGetterMockRecorder struct { + mock *MockGetter +} + +// NewMockGetter creates a new mock instance +func NewMockGetter(ctrl *gomock.Controller) *MockGetter { + mock := &MockGetter{ctrl: ctrl} + mock.recorder = &MockGetterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockGetter) EXPECT() *MockGetterMockRecorder { + return m.recorder +} + +// Get mocks base method +func (m *MockGetter) Get() (interface{}, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get") + ret0, _ := ret[0].(interface{}) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get +func (mr *MockGetterMockRecorder) Get() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockGetter)(nil).Get)) +} diff --git a/mocks/kv_mock.go b/mocks/kv_mock.go new file mode 100644 index 00000000..ad8cfa05 --- /dev/null +++ b/mocks/kv_mock.go @@ -0,0 +1,144 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: go.etcd.io/etcd/clientv3 (interfaces: KV) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + gomock "github.com/golang/mock/gomock" + clientv3 "go.etcd.io/etcd/clientv3" + reflect "reflect" +) + +// MockKV is a mock of KV interface +type MockKV struct { + ctrl *gomock.Controller + recorder *MockKVMockRecorder +} + +// MockKVMockRecorder is the mock recorder for MockKV +type MockKVMockRecorder struct { + mock *MockKV +} + +// NewMockKV creates a new mock instance +func NewMockKV(ctrl *gomock.Controller) *MockKV { + mock := &MockKV{ctrl: ctrl} + mock.recorder = &MockKVMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockKV) EXPECT() *MockKVMockRecorder { + return m.recorder +} + +// Compact mocks base method +func (m *MockKV) Compact(arg0 context.Context, arg1 int64, arg2 ...clientv3.CompactOption) (*clientv3.CompactResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Compact", varargs...) + ret0, _ := ret[0].(*clientv3.CompactResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Compact indicates an expected call of Compact +func (mr *MockKVMockRecorder) Compact(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Compact", reflect.TypeOf((*MockKV)(nil).Compact), varargs...) +} + +// Delete mocks base method +func (m *MockKV) Delete(arg0 context.Context, arg1 string, arg2 ...clientv3.OpOption) (*clientv3.DeleteResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Delete", varargs...) + ret0, _ := ret[0].(*clientv3.DeleteResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Delete indicates an expected call of Delete +func (mr *MockKVMockRecorder) Delete(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockKV)(nil).Delete), varargs...) +} + +// Do mocks base method +func (m *MockKV) Do(arg0 context.Context, arg1 clientv3.Op) (clientv3.OpResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Do", arg0, arg1) + ret0, _ := ret[0].(clientv3.OpResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Do indicates an expected call of Do +func (mr *MockKVMockRecorder) Do(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Do", reflect.TypeOf((*MockKV)(nil).Do), arg0, arg1) +} + +// Get mocks base method +func (m *MockKV) Get(arg0 context.Context, arg1 string, arg2 ...clientv3.OpOption) (*clientv3.GetResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Get", varargs...) + ret0, _ := ret[0].(*clientv3.GetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get +func (mr *MockKVMockRecorder) Get(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockKV)(nil).Get), varargs...) +} + +// Put mocks base method +func (m *MockKV) Put(arg0 context.Context, arg1, arg2 string, arg3 ...clientv3.OpOption) (*clientv3.PutResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2} + for _, a := range arg3 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Put", varargs...) + ret0, _ := ret[0].(*clientv3.PutResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Put indicates an expected call of Put +func (mr *MockKVMockRecorder) Put(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockKV)(nil).Put), varargs...) +} + +// Txn mocks base method +func (m *MockKV) Txn(arg0 context.Context) clientv3.Txn { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Txn", arg0) + ret0, _ := ret[0].(clientv3.Txn) + return ret0 +} + +// Txn indicates an expected call of Txn +func (mr *MockKVMockRecorder) Txn(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Txn", reflect.TypeOf((*MockKV)(nil).Txn), arg0) +} diff --git a/mocks/loader_mock.go b/mocks/loader_mock.go new file mode 100644 index 00000000..2f2eb8ca --- /dev/null +++ b/mocks/loader_mock.go @@ -0,0 +1,91 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./loader.go + +// Package mocks is a generated GoMock package. +package mocks + +import ( + gomock "github.com/golang/mock/gomock" + konfig "github.com/lalamove/konfig" + reflect "reflect" + time "time" +) + +// MockLoader is a mock of Loader interface +type MockLoader struct { + ctrl *gomock.Controller + recorder *MockLoaderMockRecorder +} + +// MockLoaderMockRecorder is the mock recorder for MockLoader +type MockLoaderMockRecorder struct { + mock *MockLoader +} + +// NewMockLoader creates a new mock instance +func NewMockLoader(ctrl *gomock.Controller) *MockLoader { + mock := &MockLoader{ctrl: ctrl} + mock.recorder = &MockLoaderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockLoader) EXPECT() *MockLoaderMockRecorder { + return m.recorder +} + +// Name mocks base method +func (m *MockLoader) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name +func (mr *MockLoaderMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockLoader)(nil).Name)) +} + +// Load mocks base method +func (m *MockLoader) Load(arg0 konfig.Values) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Load", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Load indicates an expected call of Load +func (mr *MockLoaderMockRecorder) Load(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Load", reflect.TypeOf((*MockLoader)(nil).Load), arg0) +} + +// MaxRetry mocks base method +func (m *MockLoader) MaxRetry() int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MaxRetry") + ret0, _ := ret[0].(int) + return ret0 +} + +// MaxRetry indicates an expected call of MaxRetry +func (mr *MockLoaderMockRecorder) MaxRetry() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MaxRetry", reflect.TypeOf((*MockLoader)(nil).MaxRetry)) +} + +// RetryDelay mocks base method +func (m *MockLoader) RetryDelay() time.Duration { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RetryDelay") + ret0, _ := ret[0].(time.Duration) + return ret0 +} + +// RetryDelay indicates an expected call of RetryDelay +func (mr *MockLoaderMockRecorder) RetryDelay() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RetryDelay", reflect.TypeOf((*MockLoader)(nil).RetryDelay)) +} diff --git a/mocks/logicalclient_mock.go b/mocks/logicalclient_mock.go new file mode 100644 index 00000000..08e09d00 --- /dev/null +++ b/mocks/logicalclient_mock.go @@ -0,0 +1,79 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./loader/klvault/vaultloader.go + +// Package mocks is a generated GoMock package. +package mocks + +import ( + gomock "github.com/golang/mock/gomock" + api "github.com/hashicorp/vault/api" + reflect "reflect" +) + +// MockLogicalClient is a mock of LogicalClient interface +type MockLogicalClient struct { + ctrl *gomock.Controller + recorder *MockLogicalClientMockRecorder +} + +// MockLogicalClientMockRecorder is the mock recorder for MockLogicalClient +type MockLogicalClientMockRecorder struct { + mock *MockLogicalClient +} + +// NewMockLogicalClient creates a new mock instance +func NewMockLogicalClient(ctrl *gomock.Controller) *MockLogicalClient { + mock := &MockLogicalClient{ctrl: ctrl} + mock.recorder = &MockLogicalClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockLogicalClient) EXPECT() *MockLogicalClientMockRecorder { + return m.recorder +} + +// Read mocks base method +func (m *MockLogicalClient) Read(key string) (*api.Secret, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Read", key) + ret0, _ := ret[0].(*api.Secret) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Read indicates an expected call of Read +func (mr *MockLogicalClientMockRecorder) Read(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Read", reflect.TypeOf((*MockLogicalClient)(nil).Read), key) +} + +// Write mocks base method +func (m *MockLogicalClient) Write(key string, data map[string]interface{}) (*api.Secret, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Write", key, data) + ret0, _ := ret[0].(*api.Secret) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Write indicates an expected call of Write +func (mr *MockLogicalClientMockRecorder) Write(key, data interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockLogicalClient)(nil).Write), key, data) +} + +// ReadWithData mocks base method +func (m *MockLogicalClient) ReadWithData(key string, data map[string][]string) (*api.Secret, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadWithData", key, data) + ret0, _ := ret[0].(*api.Secret) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReadWithData indicates an expected call of ReadWithData +func (mr *MockLogicalClientMockRecorder) ReadWithData(key, data interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadWithData", reflect.TypeOf((*MockLogicalClient)(nil).ReadWithData), key, data) +} diff --git a/mocks/parser_mock.go b/mocks/parser_mock.go new file mode 100644 index 00000000..a91325d2 --- /dev/null +++ b/mocks/parser_mock.go @@ -0,0 +1,49 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./parser/parser.go + +// Package mocks is a generated GoMock package. +package mocks + +import ( + gomock "github.com/golang/mock/gomock" + konfig "github.com/lalamove/konfig" + io "io" + reflect "reflect" +) + +// MockParser is a mock of Parser interface +type MockParser struct { + ctrl *gomock.Controller + recorder *MockParserMockRecorder +} + +// MockParserMockRecorder is the mock recorder for MockParser +type MockParserMockRecorder struct { + mock *MockParser +} + +// NewMockParser creates a new mock instance +func NewMockParser(ctrl *gomock.Controller) *MockParser { + mock := &MockParser{ctrl: ctrl} + mock.recorder = &MockParserMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockParser) EXPECT() *MockParserMockRecorder { + return m.recorder +} + +// Parse mocks base method +func (m *MockParser) Parse(arg0 io.Reader, arg1 konfig.Values) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Parse", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Parse indicates an expected call of Parse +func (mr *MockParserMockRecorder) Parse(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Parse", reflect.TypeOf((*MockParser)(nil).Parse), arg0, arg1) +} diff --git a/mocks/watcher_mock.go b/mocks/watcher_mock.go new file mode 100644 index 00000000..91572ef0 --- /dev/null +++ b/mocks/watcher_mock.go @@ -0,0 +1,103 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./watcher.go + +// Package mocks is a generated GoMock package. +package mocks + +import ( + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockWatcher is a mock of Watcher interface +type MockWatcher struct { + ctrl *gomock.Controller + recorder *MockWatcherMockRecorder +} + +// MockWatcherMockRecorder is the mock recorder for MockWatcher +type MockWatcherMockRecorder struct { + mock *MockWatcher +} + +// NewMockWatcher creates a new mock instance +func NewMockWatcher(ctrl *gomock.Controller) *MockWatcher { + mock := &MockWatcher{ctrl: ctrl} + mock.recorder = &MockWatcherMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockWatcher) EXPECT() *MockWatcherMockRecorder { + return m.recorder +} + +// Start mocks base method +func (m *MockWatcher) Start() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Start") + ret0, _ := ret[0].(error) + return ret0 +} + +// Start indicates an expected call of Start +func (mr *MockWatcherMockRecorder) Start() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockWatcher)(nil).Start)) +} + +// Done mocks base method +func (m *MockWatcher) Done() <-chan struct{} { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Done") + ret0, _ := ret[0].(<-chan struct{}) + return ret0 +} + +// Done indicates an expected call of Done +func (mr *MockWatcherMockRecorder) Done() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Done", reflect.TypeOf((*MockWatcher)(nil).Done)) +} + +// Watch mocks base method +func (m *MockWatcher) Watch() <-chan struct{} { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Watch") + ret0, _ := ret[0].(<-chan struct{}) + return ret0 +} + +// Watch indicates an expected call of Watch +func (mr *MockWatcherMockRecorder) Watch() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Watch", reflect.TypeOf((*MockWatcher)(nil).Watch)) +} + +// Close mocks base method +func (m *MockWatcher) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close +func (mr *MockWatcherMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockWatcher)(nil).Close)) +} + +// Err mocks base method +func (m *MockWatcher) Err() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Err") + ret0, _ := ret[0].(error) + return ret0 +} + +// Err indicates an expected call of Err +func (mr *MockWatcherMockRecorder) Err() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Err", reflect.TypeOf((*MockWatcher)(nil).Err)) +} diff --git a/parser/kpjson/README.md b/parser/kpjson/README.md new file mode 100644 index 00000000..ca6ee7be --- /dev/null +++ b/parser/kpjson/README.md @@ -0,0 +1,29 @@ +# JSON Parser +JSON parser parses a JSON file to a map[string]interface{} and then traverses the map and adds values into the config store flattening the JSON using dot notation for keys. + +Ex: +``` +{ + "foo": "bar", + "nested": { + "firstName": "john", + "lastName": "doe", + "list": [ + 1, + 2, + ] + } +} +``` +Will add the following key/value to the config +``` +"foo" => "bar" +"nested.firstName" => "john" +"nested.lastName" => "doe" +"nested.list" => []int{1,2} +``` + +# Usage +``` +err := kpjson.Parser.Parse(strings.NewReader(`{"foo":"bar"}`), konfig.Values{}) +``` diff --git a/parser/kpjson/jsonparser.go b/parser/kpjson/jsonparser.go new file mode 100644 index 00000000..51f04628 --- /dev/null +++ b/parser/kpjson/jsonparser.go @@ -0,0 +1,26 @@ +package kpjson + +import ( + "encoding/json" + "io" + + "github.com/lalamove/konfig" + "github.com/lalamove/konfig/parser" + "github.com/lalamove/konfig/parser/kpmap" +) + +// Parser parses the given json io.Reader and adds values in dot.path notation into the konfig.Store +var Parser = parser.Func(func(r io.Reader, s konfig.Values) error { + // unmarshal the JSON into map[string]interface{} + var dec = json.NewDecoder(r) + + var d = make(map[string]interface{}) + var err = dec.Decode(&d) + if err != nil { + return err + } + + kpmap.PopFlatten(d, s) + + return nil +}) diff --git a/parser/kpjson/jsonparser_test.go b/parser/kpjson/jsonparser_test.go new file mode 100644 index 00000000..cc5b3b2f --- /dev/null +++ b/parser/kpjson/jsonparser_test.go @@ -0,0 +1,109 @@ +package kpjson + +import ( + "strings" + "testing" + + "github.com/lalamove/konfig" + "github.com/stretchr/testify/require" +) + +func TestJSONParser(t *testing.T) { + var testCases = []struct { + name string + json string + asserts func(t *testing.T, s konfig.Values) + }{ + { + name: "simple 1 level json object", + json: `{"foo":"bar","bar":"foo","int":1}`, + asserts: func(t *testing.T, v konfig.Values) { + require.Equal( + t, + "bar", + v["foo"], + ) + + require.Equal( + t, + "foo", + v["bar"], + ) + + require.Equal( + t, + float64(1), + v["int"], + ) + }, + }, + { + name: "nested objects", + json: `{ + "foo": "bar", + "bar": { + "foo": "hello world!", + "bool": true, + "nested": { + "john": "doe" + } + } + }`, + asserts: func(t *testing.T, v konfig.Values) { + require.Equal( + t, + "bar", + v["foo"], + ) + + require.Equal( + t, + "hello world!", + v["bar.foo"], + ) + + require.Equal( + t, + true, + v["bar.bool"], + ) + + require.Equal( + t, + "doe", + v["bar.nested.john"], + ) + }, + }, + } + + for _, testCase := range testCases { + t.Run( + testCase.name, + func(t *testing.T) { + konfig.Init(konfig.DefaultConfig()) + + var v = konfig.Values{} + var err = Parser.Parse( + strings.NewReader( + testCase.json, + ), + v, + ) + + require.Nil(t, err) + testCase.asserts(t, v) + }, + ) + } +} + +func TestParserErr(t *testing.T) { + var err = Parser.Parse( + strings.NewReader( + `invalid`, + ), + konfig.Values{}, + ) + require.NotNil(t, err) +} diff --git a/parser/kpkeyval/kvparser.go b/parser/kpkeyval/kvparser.go new file mode 100644 index 00000000..44c9e90c --- /dev/null +++ b/parser/kpkeyval/kvparser.go @@ -0,0 +1,63 @@ +// Package kpkeyval provides a key value parser to parse an io.Reader's content +// of key/values with a configurable separator and add it into a konfig.Store. +package kpkeyval + +import ( + "bufio" + "errors" + "io" + "strings" + + "github.com/lalamove/konfig" + "github.com/lalamove/konfig/parser" +) + +// DefaultSep is the default key value separator +const DefaultSep = "=" + +// ErrInvalidConfigFileFormat is the error returned when a problem is encountered when parsing the +// config file +var ( + ErrInvalidConfigFileFormat = errors.New("Err invalid file format") + // make sure Parser implements fileloader.Parser + _ parser.Parser = (*Parser)(nil) +) + +// Config is the configuration of the key value parser +type Config struct { + // Sep is the separator between keys and values + Sep string +} + +// Parser implements fileloader.Parser +// It parses a file of key/values with a specific separator +// and stores in the konfig.Store +type Parser struct { + cfg *Config +} + +// New creates a new parser with the given config +func New(cfg *Config) *Parser { + if cfg.Sep == "" { + cfg.Sep = DefaultSep + } + return &Parser{ + cfg: cfg, + } +} + +// Parse implement the fileloader.Parser interface +func (k *Parser) Parse(r io.Reader, cfg konfig.Values) error { + var scanner = bufio.NewScanner(r) + for scanner.Scan() { + var cfgKey = strings.Split(scanner.Text(), k.cfg.Sep) + if len(cfgKey) < 2 { + return ErrInvalidConfigFileFormat + } + cfg.Set(cfgKey[0], strings.Join(cfgKey[1:], k.cfg.Sep)) + } + if err := scanner.Err(); err != nil { + return err + } + return nil +} diff --git a/parser/kpkeyval/kvparser_test.go b/parser/kpkeyval/kvparser_test.go new file mode 100644 index 00000000..a8240623 --- /dev/null +++ b/parser/kpkeyval/kvparser_test.go @@ -0,0 +1,90 @@ +package kpkeyval + +import ( + "errors" + "io" + "strings" + "testing" + + "github.com/lalamove/konfig" + "github.com/stretchr/testify/require" +) + +type ErrReader struct{} + +func (e ErrReader) Read([]byte) (int, error) { + return 0, errors.New("") +} + +func TestKVParser(t *testing.T) { + var testCases = []struct { + name string + err bool + reader io.Reader + sep string + asserts func(t *testing.T, cfg konfig.Values) + }{ + { + name: `no error, default separator`, + err: false, + reader: strings.NewReader("BAR=FOO\nFOO=BAR"), + asserts: func(t *testing.T, cfg konfig.Values) { + require.Equal(t, "BAR", cfg["FOO"]) + require.Equal(t, "FOO", cfg["BAR"]) + }, + }, + { + name: `no error, custom separator`, + err: false, + sep: ":", + reader: strings.NewReader("BAR:FOO\nFOO:BAR"), + asserts: func(t *testing.T, cfg konfig.Values) { + require.Equal(t, "BAR", cfg["FOO"]) + require.Equal(t, "FOO", cfg["BAR"]) + }, + }, + { + name: `err invalid format`, + err: true, + sep: ":", + reader: strings.NewReader("BAR\nFOO"), + }, + { + name: `err scanner`, + err: true, + sep: ":", + reader: ErrReader{}, + }, + } + + for _, testCase := range testCases { + t.Run( + testCase.name, + func(t *testing.T) { + var v = konfig.Values{} + var p = New(&Config{ + Sep: testCase.sep, + }) + + var err = p.Parse(testCase.reader, v) + if testCase.err { + require.NotNil(t, err, "err should not be nil") + return + } + require.Nil(t, err, "err should be nil") + + testCase.asserts(t, v) + }, + ) + } +} + +func TestParserErr(t *testing.T) { + var err = New(&Config{}).Parse( + strings.NewReader( + `invalid`, + ), + konfig.Values{}, + ) + require.NotNil(t, err) +} diff --git a/parser/kpmap/mapparser.go b/parser/kpmap/mapparser.go new file mode 100644 index 00000000..ca31cc0b --- /dev/null +++ b/parser/kpmap/mapparser.go @@ -0,0 +1,39 @@ +package kpmap + +import ( + "fmt" + + "github.com/lalamove/konfig" +) + +func traverseMapIface(m map[interface{}]interface{}, s konfig.Values, p string) { + for k, v := range m { + var ks = fmt.Sprintf("%v", k) + switch vt := v.(type) { + case map[string]interface{}: + traverseMap(vt, s, p+ks+konfig.KeySep) + case map[interface{}]interface{}: + traverseMapIface(vt, s, p+ks+konfig.KeySep) + default: + s.Set(p+ks, v) + } + } +} + +func traverseMap(m map[string]interface{}, s konfig.Values, p string) { + for k, v := range m { + switch vt := v.(type) { + case map[string]interface{}: + traverseMap(vt, s, p+k+konfig.KeySep) + case map[interface{}]interface{}: + traverseMapIface(vt, s, p+k+konfig.KeySep) + default: + s.Set(p+k, v) + } + } +} + +// PopFlatten populates a konfig.Store by flatteing a map[string]interface{} +func PopFlatten(m map[string]interface{}, s konfig.Values) { + traverseMap(m, s, "") +} diff --git a/parser/kpmap/mapparser_test.go b/parser/kpmap/mapparser_test.go new file mode 100644 index 00000000..fec1a260 --- /dev/null +++ b/parser/kpmap/mapparser_test.go @@ -0,0 +1 @@ +package kpmap diff --git a/parser/kptoml/README.md b/parser/kptoml/README.md new file mode 100644 index 00000000..6035eda6 --- /dev/null +++ b/parser/kptoml/README.md @@ -0,0 +1,24 @@ +# TOML Parser +TOML parser parses a TOML file to a map[string]interface{} and then traverses the map and adds values into the config store flattening the TOML using dot notation for keys. + +Ex: +``` +foo = bar + +[nested] +firstName = "john" +lastName = "doe" +list = [1, 2] +``` +Will add the following key/value to the config +``` +"foo" => "bar" +"nested.firstName" => "john" +"nested.lastName" => "doe" +"nested.list" => []int{1,2} +``` + +# Usage +``` +err := kptoml.Parser.Parse(strings.NewReader(`foo: "bar"`), konfig.Values{}) +``` diff --git a/parser/kptoml/tomlparser.go b/parser/kptoml/tomlparser.go new file mode 100644 index 00000000..567c208f --- /dev/null +++ b/parser/kptoml/tomlparser.go @@ -0,0 +1,24 @@ +package kptoml + +import ( + "io" + + "github.com/lalamove/konfig" + "github.com/lalamove/konfig/parser" + "github.com/lalamove/konfig/parser/kpmap" + "github.com/BurntSushi/toml" +) + +// Parser parses the given json io.Reader and adds values in dot.path notation into the konfig.Store +var Parser = parser.Func(func(r io.Reader, s konfig.Values) error { + // unmarshal the JSON into map[string]interface{} + var d = make(map[string]interface{}) + var _, err = toml.DecodeReader(r, &d) + if err != nil { + return err + } + + kpmap.PopFlatten(d, s) + + return nil +}) diff --git a/parser/kptoml/tomlparser_test.go b/parser/kptoml/tomlparser_test.go new file mode 100644 index 00000000..88542cb1 --- /dev/null +++ b/parser/kptoml/tomlparser_test.go @@ -0,0 +1,113 @@ +package kptoml + +import ( + "strings" + "testing" + + "github.com/lalamove/konfig" + "github.com/stretchr/testify/require" +) + +func TestJSONParser(t *testing.T) { + var testCases = []struct { + name string + toml string + asserts func(t *testing.T, v konfig.Values) + }{ + { + name: "simple 1 level toml object", + toml: ` +foo = "bar" +bar = "foo" +int = 1 +`, + asserts: func(t *testing.T, v konfig.Values) { + require.Equal( + t, + "bar", + v["foo"], + ) + + require.Equal( + t, + "foo", + v["bar"], + ) + + require.Equal( + t, + int64(1), + v["int"], + ) + }, + }, + { + name: "nested objects", + toml: ` +foo = "bar" + +[bar] +foo = "hello world!" +bool = true + +[bar.nested] +john = "doe" +`, + asserts: func(t *testing.T, v konfig.Values) { + require.Equal( + t, + "bar", + v["foo"], + ) + + require.Equal( + t, + "hello world!", + v["bar.foo"], + ) + + require.Equal( + t, + true, + v["bar.bool"], + ) + + require.Equal( + t, + "doe", + v["bar.nested.john"], + ) + }, + }, + } + + for _, testCase := range testCases { + t.Run( + testCase.name, + func(t *testing.T) { + konfig.Init(konfig.DefaultConfig()) + + var v = konfig.Values{} + var err = Parser.Parse( + strings.NewReader( + testCase.toml, + ), + v, + ) + + require.Nil(t, err) + testCase.asserts(t, v) + }, + ) + } +} + +func TestParserErr(t *testing.T) { + var err = Parser.Parse( + strings.NewReader( + `invalid`, + ), + konfig.Values{}, + ) + require.NotNil(t, err) +} diff --git a/parser/kpyaml/README.md b/parser/kpyaml/README.md new file mode 100644 index 00000000..45536217 --- /dev/null +++ b/parser/kpyaml/README.md @@ -0,0 +1,25 @@ +# YAML Parser +YAML parser parses a YAML file to a map[string]interface{} and then traverses the map and adds values into the config store flattening the YAML using dot notation for keys. + +Ex: +``` +foo: "bar" +nested: + firstName: "john" + lastName: "doe" + list: + - 1 + - 2 +``` +Will add the following key/value to the config +``` +"foo" => "bar" +"nested.firstName" => "john" +"nested.lastName" => "doe" +"nested.list" => []int{1,2} +``` + +# Usage +``` +err := kpyaml.Parser.Parse(strings.NewReader(`foo: "bar"`), konfig.Values{}) +``` diff --git a/parser/kpyaml/yamlparser.go b/parser/kpyaml/yamlparser.go new file mode 100644 index 00000000..83645032 --- /dev/null +++ b/parser/kpyaml/yamlparser.go @@ -0,0 +1,25 @@ +package kpyaml + +import ( + "io" + + "github.com/lalamove/konfig" + "github.com/lalamove/konfig/parser" + "github.com/lalamove/konfig/parser/kpmap" + yaml "gopkg.in/yaml.v2" +) + +// Parser is the YAML Parser it implements parser.Parser +var Parser = parser.Func(func(r io.Reader, s konfig.Values) error { + var dec = yaml.NewDecoder(r) + + var d = make(map[string]interface{}) + var err = dec.Decode(&d) + if err != nil { + return err + } + + kpmap.PopFlatten(d, s) + + return nil +}) diff --git a/parser/kpyaml/yamlparser_test.go b/parser/kpyaml/yamlparser_test.go new file mode 100644 index 00000000..8e1e00de --- /dev/null +++ b/parser/kpyaml/yamlparser_test.go @@ -0,0 +1,119 @@ +package kpyaml + +import ( + "strings" + "testing" + + "github.com/lalamove/konfig" + "github.com/stretchr/testify/require" +) + +func TestYAMLParser(t *testing.T) { + var testCases = []struct { + name string + yaml string + asserts func(t *testing.T, v konfig.Values) + }{ + { + name: "simple 1 level yaml object", + yaml: `foo: bar +bar: foo +int: 1`, + asserts: func(t *testing.T, v konfig.Values) { + require.Equal( + t, + "bar", + v["foo"], + ) + + require.Equal( + t, + "foo", + v["bar"], + ) + + require.Equal( + t, + 1, + v["int"], + ) + }, + }, + { + name: "nested objects", + yaml: `foo: bar +bar: + foo: "hello world!" + bool: true + nested: + john: "doe"`, + asserts: func(t *testing.T, v konfig.Values) { + require.Equal( + t, + "bar", + v["foo"], + ) + + require.Equal( + t, + "hello world!", + v["bar.foo"], + ) + + require.Equal( + t, + true, + v["bar.bool"], + ) + + require.Equal( + t, + "doe", + v["bar.nested.john"], + ) + }, + }, + { + name: "nested objects", + yaml: `foo: bar +bar: + - "hello world!" + - "yaml"`, + asserts: func(t *testing.T, v konfig.Values) { + require.Equal( + t, + []interface{}{"hello world!", "yaml"}, + v["bar"], + ) + }, + }, + } + + for _, testCase := range testCases { + t.Run( + testCase.name, + func(t *testing.T) { + konfig.Init(konfig.DefaultConfig()) + var v = konfig.Values{} + var err = Parser.Parse( + strings.NewReader( + testCase.yaml, + ), + v, + ) + require.Nil(t, err) + testCase.asserts(t, v) + }, + ) + } +} + +func TestParserErr(t *testing.T) { + var err = Parser.Parse( + strings.NewReader( + `invalid`, + ), + konfig.Values{}, + ) + require.NotNil(t, err) +} diff --git a/parser/parser.go b/parser/parser.go new file mode 100644 index 00000000..bdaabed1 --- /dev/null +++ b/parser/parser.go @@ -0,0 +1,22 @@ +package parser + +import ( + "io" + + "github.com/lalamove/konfig" +) + +var _ Parser = (Func)(nil) + +// Parser is the interface to implement to parse a config file +type Parser interface { + Parse(io.Reader, konfig.Values) error +} + +// Func is a function implementing the Parser interface +type Func func(io.Reader, konfig.Values) error + +// Parse implements Parser interface +func (f Func) Parse(r io.Reader, s konfig.Values) error { + return f(r, s) +} diff --git a/parser/parser_test.go b/parser/parser_test.go new file mode 100644 index 00000000..c372228c --- /dev/null +++ b/parser/parser_test.go @@ -0,0 +1,19 @@ +package parser + +import ( + io "io" + "testing" + + "github.com/lalamove/konfig" + "github.com/stretchr/testify/require" +) + +func TestParserFunc(t *testing.T) { + var ran bool + var f = Func(func(r io.Reader, s konfig.Values) error { + ran = true + return nil + }) + f.Parse(nil, nil) + require.Equal(t, true, ran) +} diff --git a/test.sh b/test.sh new file mode 100755 index 00000000..3c3bf9d4 --- /dev/null +++ b/test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e +echo "" > coverage.txt + +for d in $(go list ./... | grep -v vendor); do + go test -race -v -coverprofile=profile.out -covermode=atomic $d + if [ -f profile.out ]; then + cat profile.out >> coverage.txt + rm profile.out + fi +done diff --git a/test/configfile_test.go b/test/configfile_test.go new file mode 100644 index 00000000..ee9ddaf8 --- /dev/null +++ b/test/configfile_test.go @@ -0,0 +1,89 @@ +package test + +import ( + "testing" + "time" + + "github.com/lalamove/konfig" + "github.com/lalamove/konfig/loader/klfile" + "github.com/lalamove/konfig/parser/kpyaml" + "github.com/stretchr/testify/require" +) + +type DBConfig struct { + MySQL MySQLConfig `konfig:"mysql"` + Redis RedisConfig `konfig:"redis"` +} + +type RedisConfig struct { + H string `konfig:"host"` +} + +type MySQLConfig struct { + U string `konfig:"username"` + PW string `konfig:"password"` + MaxOpenConn int +} + +type VaultConfig struct { + Enable bool + Server string + Secret string `konfig:"dbSecret"` +} + +type YAMLConfig struct { + Debug bool `konfig:"debug"` + SQLDebug bool `konfig:"sqlDebug"` + DB DBConfig + Port string `konfig:"http.port"` + Vault VaultConfig +} + +func TestYAMLFile(t *testing.T) { + var expectedConfig = YAMLConfig{ + Debug: true, + SQLDebug: true, + Port: "8081", + DB: DBConfig{ + MySQL: MySQLConfig{ + U: "username", + PW: "password", + MaxOpenConn: 10, + }, + Redis: RedisConfig{ + H: "127.0.0.1", + }, + }, + Vault: VaultConfig{ + Enable: true, + Server: "http://127.0.0.1:8200", + Secret: "/secret/db1", + }, + } + + konfig.Init(&konfig.Config{ + NoExitOnError: true, + }) + + konfig.Bind(YAMLConfig{}) + + konfig.RegisterLoader( + klfile.New(&klfile.Config{ + Files: []klfile.File{ + klfile.File{ + Path: "./data/cfg.yml", + Parser: kpyaml.Parser, + }, + }, + MaxRetry: 2, + RetryDelay: 1 * time.Second, + Debug: true, + }), + ) + + if err := konfig.Load(); err != nil { + t.Error(err) + } + + require.Equal(t, expectedConfig, konfig.Value()) +} diff --git a/test/data/cfg b/test/data/cfg new file mode 100644 index 00000000..01077e73 --- /dev/null +++ b/test/data/cfg @@ -0,0 +1,3 @@ +KEY=VALUE +KEY1=VALUE1 +KEY2=VALUE2=efwefwef diff --git a/test/data/cfg.yml b/test/data/cfg.yml new file mode 100644 index 00000000..6ff20bf6 --- /dev/null +++ b/test/data/cfg.yml @@ -0,0 +1,15 @@ +debug: true +sqlDebug: true +vault: + enable: true + server: "http://127.0.0.1:8200" + dbSecret: "/secret/db1" +db: + mysql: + username: "username" + password: "password" + maxOpenConn: 10 + redis: + host: "127.0.0.1" +http: + port: "8081" diff --git a/util.go b/util.go new file mode 100644 index 00000000..d57cc1e1 --- /dev/null +++ b/util.go @@ -0,0 +1,264 @@ +package konfig + +import ( + "fmt" + "time" + + "github.com/spf13/cast" +) + +type s map[string]interface{} + +// Exists checks if a config key k is set in the Store +func Exists(k string) bool { + return instance().Exists(k) +} +func (c *store) Exists(k string) bool { + var m = c.m.Load().(s) + _, ok := m[k] + return ok +} + +// Get will return the value in config with given key k +// If not value is found, Get it returns nil +func Get(k string) interface{} { + return instance().Get(k) +} + +// MustGet returns the value in config with given key k +// If not found it panics +func MustGet(k string) interface{} { + return instance().MustGet(k) +} + +// Set will set the key value to the sync.Map +func Set(k string, v interface{}) { + instance().Set(k, v) +} + +// Set sets a value in config +func (c *store) Set(k string, v interface{}) { + c.mut.Lock() + defer c.mut.Unlock() + + var m = c.m.Load().(s) + + var nm = make(s) + for kk, vv := range m { + nm[kk] = vv + } + nm[k] = v + + // if there is a value bound we set it there also + if c.v != nil { + c.v.set(k, v) + } + + c.m.Store(nm) +} + +// Get gets a value from config +func (c *store) Get(k string) interface{} { + var m = c.m.Load().(s) + if v, ok := m[k]; ok { + return v + } + return nil +} + +// MustGet gets a value from config and panics if the value does not exist +func (c *store) MustGet(k string) interface{} { + var m = c.m.Load().(s) + if v, ok := m[k]; ok { + return v + } + panic(fmt.Errorf(ErrConfigNotFoundMsg, k)) +} + +// MustInt gets the config k and tries to convert it to an int +// it panics if the config does not exist or it fails to convert it to an int. +func MustInt(k string) int { + return instance().MustInt(k) +} + +func (c *store) MustInt(k string) int { + return cast.ToInt(c.MustGet(k)) +} + +// Int gets the config k and tries to convert it to an int +// It returns the zero value if it doesn't find the config. +func Int(k string) int { + return instance().Int(k) +} +func (c *store) Int(k string) int { + return cast.ToInt(c.Get(k)) +} + +// MustFloat gets the config k and tries to convert it to a float64 +// it panics if it fails. +func MustFloat(k string) float64 { + return instance().MustFloat(k) +} +func (c *store) MustFloat(k string) float64 { + return cast.ToFloat64(c.MustGet(k)) +} + +// Float gets the config k and tries to convert it to float64 +// It returns the zero value if it doesn't find the config. +func Float(k string) float64 { + return instance().Float(k) +} +func (c *store) Float(k string) float64 { + return cast.ToFloat64(c.Get(k)) +} + +// MustString gets the config k and tries to convert it to a string +// it panics if it fails. +func MustString(k string) string { + return instance().MustString(k) +} + +func (c *store) MustString(k string) string { + return cast.ToString(c.MustGet(k)) +} + +// String gets the config k and tries to convert it to a string +// It returns the zero value if it doesn't find the config. +func String(k string) string { + return instance().String(k) +} +func (c *store) String(k string) string { + return cast.ToString(c.Get(k)) +} + +// MustBool gets the config k and tries to convert it to a bool +// it panics if it fails. +func MustBool(k string) bool { + return instance().MustBool(k) +} + +func (c *store) MustBool(k string) bool { + return cast.ToBool(c.MustGet(k)) +} + +// Bool gets the config k and converts it to a bool. +// It returns the zero value if it doesn't find the config. +func Bool(k string) bool { + return instance().Bool(k) +} + +func (c *store) Bool(k string) bool { + return cast.ToBool(c.Get(k)) +} + +// MustDuration gets the config k and tries to convert it to a duration +// it panics if it fails. +func MustDuration(k string) time.Duration { + return instance().MustDuration(k) +} + +func (c *store) MustDuration(k string) time.Duration { + return cast.ToDuration(c.MustGet(k)) +} + +// Duration gets the config k and converts it to a duration. +// It returns the zero value if it doesn't find the config. +func Duration(k string) time.Duration { + return instance().Duration(k) +} + +func (c *store) Duration(k string) time.Duration { + return cast.ToDuration(c.Get(k)) +} + +// MustTime gets the config k and tries to convert it to a time.Time +// it panics if it fails. +func MustTime(k string) time.Time { + return instance().MustTime(k) +} + +func (c *store) MustTime(k string) time.Time { + return cast.ToTime(c.MustGet(k)) +} + +// Time gets the config k and converts it to a time.Time. +// It returns the zero value if it doesn't find the config. +func Time(k string) time.Time { + return instance().Time(k) +} + +func (c *store) Time(k string) time.Time { + return cast.ToTime(c.Get(k)) +} + +// MustStringSlice gets the config k and tries to convert it to a []string +// it panics if it fails. +func MustStringSlice(k string) []string { + return instance().MustStringSlice(k) +} +func (c *store) MustStringSlice(k string) []string { + return cast.ToStringSlice(c.MustGet(k)) +} + +// StringSlice gets the config k and converts it to a []string. +// It returns the zero value if it doesn't find the config. +func StringSlice(k string) []string { + return instance().StringSlice(k) +} +func (c *store) StringSlice(k string) []string { + return cast.ToStringSlice(c.Get(k)) +} + +// MustIntSlice gets the config k and tries to convert it to a []int +// it panics if it fails. +func MustIntSlice(k string) []int { + return instance().MustIntSlice(k) +} +func (c *store) MustIntSlice(k string) []int { + return cast.ToIntSlice(c.MustGet(k)) +} + +// IntSlice gets the config k and converts it to a []int. +// it returns the zero value if it doesn't find the config. +func IntSlice(k string) []int { + return instance().IntSlice(k) +} +func (c *store) IntSlice(k string) []int { + return cast.ToIntSlice(c.Get(k)) +} + +// MustStringMap gets the config k and tries to convert it to a map[string]interface{} +// it panics if it fails. +func MustStringMap(k string) map[string]interface{} { + return instance().MustStringMap(k) +} +func (c *store) MustStringMap(k string) map[string]interface{} { + return cast.ToStringMap(c.MustGet(k)) +} + +// StringMap gets the config k and converts it to a map[string]interface{}. +// it returns the zero value if it doesn't find the config. +func StringMap(k string) map[string]interface{} { + return instance().StringMap(k) +} +func (c *store) StringMap(k string) map[string]interface{} { + return cast.ToStringMap(c.Get(k)) +} + +// MustStringMapString gets the config k and tries to convert it to a map[string]string +// it panics if it fails. +func MustStringMapString(k string) map[string]string { + return instance().MustStringMapString(k) +} +func (c *store) MustStringMapString(k string) map[string]string { + return cast.ToStringMapString(c.MustGet(k)) +} + +// StringMapString gets the config k and converts it to a map[string]string. +// it returns the zero value if it doesn't find the config. +func StringMapString(k string) map[string]string { + return instance().StringMapString(k) +} +func (c *store) StringMapString(k string) map[string]string { + return cast.ToStringMapString(c.Get(k)) +} diff --git a/util_test.go b/util_test.go new file mode 100644 index 00000000..7e2fff67 --- /dev/null +++ b/util_test.go @@ -0,0 +1,269 @@ +package konfig + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestUtils(t *testing.T) { + var testCases = []struct { + name string + test func(t *testing.T) + }{ + { + name: "IntSuccess", + test: func(t *testing.T) { + Set("foo", 1) + var i = Int("foo") + require.Equal(t, 1, i) + }, + }, + { + name: "MustIntSuccess", + test: func(t *testing.T) { + Set("foo", 1) + var i int + require.NotPanics(t, func() { + i = MustInt("foo") + }) + require.Equal(t, 1, i) + }, + }, + { + name: "MustIntPanic", + test: func(t *testing.T) { + require.Panics(t, func() { + MustInt("foo") + }) + }, + }, + { + name: "StringSuccess", + test: func(t *testing.T) { + Set("foo", "bar") + var s = String("foo") + require.Equal(t, "bar", s) + }, + }, + { + name: "MustStringSuccess", + test: func(t *testing.T) { + Set("foo", "bar") + var s string + require.NotPanics(t, func() { s = MustString("foo") }) + require.Equal(t, "bar", s) + }, + }, + { + + name: "MustStringPanics", + test: func(t *testing.T) { + require.Panics(t, func() { MustString("foo") }) + }, + }, + { + + name: "FloatSuccess", + test: func(t *testing.T) { + Set("foo", 2.1) + var f = Float("foo") + require.Equal(t, 2.1, f) + }, + }, + { + name: "MustFloatSuccess", + test: func(t *testing.T) { + Set("foo", 1.1) + var f float64 + require.NotPanics(t, func() { f = MustFloat("foo") }) + require.Equal(t, 1.1, f) + }, + }, + { + name: "MustFloatPanics", + test: func(t *testing.T) { + require.Panics(t, func() { MustFloat("foo") }) + }, + }, + + { + name: "BoolSuccess", + test: func(t *testing.T) { + Set("foo", true) + var b = Bool("foo") + require.Equal(t, true, b) + }, + }, + { + name: "MustBoolSuccess", + test: func(t *testing.T) { + Set("foo", true) + var b bool + require.NotPanics(t, func() { b = MustBool("foo") }) + require.Equal(t, true, b) + }, + }, + { + name: "MustBoolPanics", + test: func(t *testing.T) { + require.Panics(t, func() { MustBool("foo") }) + }, + }, + + { + name: "DurationSuccess", + test: func(t *testing.T) { + Set("foo", "1m") + var d = Duration("foo") + require.Equal(t, 1*time.Minute, d) + }, + }, + { + name: "MustDurationSuccess", + test: func(t *testing.T) { + Set("foo", "1m") + var d time.Duration + require.NotPanics(t, func() { d = MustDuration("foo") }) + require.Equal(t, 1*time.Minute, d) + }, + }, + { + name: "MustDurationPanics", + test: func(t *testing.T) { + require.Panics(t, func() { MustDuration("foo") }) + }, + }, + { + name: "TimeSuccess", + test: func(t *testing.T) { + Set("foo", "2019-01-02T15:04:05Z07:00") + var d = Time("foo") + + var ti, _ = time.Parse(time.RFC3339, "2019-01-02T15:04:05Z07:00") + + require.Equal(t, ti, d) + }, + }, + { + name: "MustTimeSuccess", + test: func(t *testing.T) { + Set("foo", "2019-01-02T15:04:05Z07:00") + var d time.Time + + var ti, _ = time.Parse(time.RFC3339, "2019-01-02T15:04:05Z07:00") + + require.NotPanics(t, func() { d = MustTime("foo") }) + + require.Equal(t, ti, d) + }, + }, + { + name: "MustTimePanics", + test: func(t *testing.T) { + require.Panics(t, func() { MustTime("foo") }) + }, + }, + + { + name: "StringSliceSuccess", + test: func(t *testing.T) { + Set("foo", []string{"bar"}) + var b = StringSlice("foo") + require.Equal(t, []string{"bar"}, b) + }, + }, + { + name: "MustStringSliceSuccess", + test: func(t *testing.T) { + Set("foo", []string{"bar"}) + var b []string + require.NotPanics(t, func() { b = MustStringSlice("foo") }) + require.Equal(t, []string{"bar"}, b) + }, + }, + { + name: "MustStringSlicePanics", + test: func(t *testing.T) { + require.Panics(t, func() { MustStringSlice("foo") }) + }, + }, + { + name: "IntSliceSuccess", + test: func(t *testing.T) { + Set("foo", []int{1}) + var b = IntSlice("foo") + require.Equal(t, []int{1}, b) + }, + }, + { + name: "MustIntSliceSuccess", + test: func(t *testing.T) { + Set("foo", []int{1}) + var b []int + require.NotPanics(t, func() { b = MustIntSlice("foo") }) + require.Equal(t, []int{1}, b) + }, + }, + { + name: "MustIntSlicePanics", + test: func(t *testing.T) { + require.Panics(t, func() { MustIntSlice("foo") }) + }, + }, + { + name: "StringMapSuccess", + test: func(t *testing.T) { + Set("foo", map[string]interface{}{"foo": "bar"}) + var b = StringMap("foo") + require.Equal(t, map[string]interface{}{"foo": "bar"}, b) + }, + }, + { + name: "MustStringMapSuccess", + test: func(t *testing.T) { + Set("foo", map[string]interface{}{"foo": "bar"}) + var b map[string]interface{} + require.NotPanics(t, func() { b = MustStringMap("foo") }) + require.Equal(t, map[string]interface{}{"foo": "bar"}, b) + }, + }, + { + name: "MustStringMapPanics", + test: func(t *testing.T) { + require.Panics(t, func() { MustStringMap("foo") }) + }, + }, + { + name: "StringMapStringSuccess", + test: func(t *testing.T) { + Set("foo", map[string]string{"foo": "bar"}) + var b = StringMapString("foo") + require.Equal(t, map[string]string{"foo": "bar"}, b) + }, + }, + { + name: "MustStringMapStringSuccess", + test: func(t *testing.T) { + Set("foo", map[string]string{"foo": "bar"}) + var b map[string]string + require.NotPanics(t, func() { b = MustStringMapString("foo") }) + require.Equal(t, map[string]string{"foo": "bar"}, b) + }, + }, + { + name: "MustStringMapStringPanics", + test: func(t *testing.T) { + require.Panics(t, func() { MustStringMapString("foo") }) + }, + }, + } + for _, testCase := range testCases { + + t.Run(testCase.name, func(t *testing.T) { + reset() + testCase.test(t) + }) + } +} diff --git a/value.go b/value.go new file mode 100644 index 00000000..e3794819 --- /dev/null +++ b/value.go @@ -0,0 +1,288 @@ +package konfig + +import ( + "errors" + "fmt" + "reflect" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/jinzhu/copier" + "github.com/spf13/cast" +) + +const ( + // TagKey is the tag key to unmarshal config values to bound value + TagKey = "konfig" + // KeySep is the separator for config keys + KeySep = "." +) + +var ( + // ErrIncorrectValue is the error thrown when trying to bind an invalid type to a config store + ErrIncorrectValue = errors.New("Bind takes a map[string]interface{} or a struct") +) + +type value struct { + s *store + v *atomic.Value + vt reflect.Type + mut *sync.Mutex + isMap bool +} + +// Value returns the value bound to the root config store +func Value() interface{} { + return instance().Value() +} + +// Bind binds a value to the root config store +func Bind(v interface{}) { + instance().Bind(v) +} + +// Value returns the value bound to the config store +func (c *store) Value() interface{} { + return c.v.v.Load() +} + +// Bind binds a value (either a map[string]interface{} or a struct) to the config store. When config values are set on the config store, they are also set on the bound value. +func (c *store) Bind(v interface{}) { + var t = reflect.TypeOf(v) + var k = t.Kind() + // if it is neither a map nor a struct + if k != reflect.Map && k != reflect.Struct { + panic(ErrIncorrectValue) + } + // if it is a map check map[string]interface{} + if k == reflect.Map && + (t.Key().Kind() != reflect.String || t.Elem().Kind() != reflect.Interface) { + panic(ErrIncorrectValue) + } + + var val = &value{ + s: c, + isMap: k == reflect.Map, + mut: &sync.Mutex{}, + } + + val.vt = t + + // create a new pointer to the given value and store it + var atomicValue atomic.Value + var n = reflect.Zero(val.vt) + atomicValue.Store(n.Interface()) + + val.v = &atomicValue + + c.v = val +} + +func (val *value) set(k string, v interface{}) { + val.mut.Lock() + defer val.mut.Unlock() + + var configValue = val.v.Load() + + // if value is a map + // store things in a map + if val.isMap { + var mapV = configValue.(map[string]interface{}) + var nMap = make(map[string]interface{}) + + for kk, vv := range mapV { + nMap[kk] = vv + } + + nMap[k] = v + + val.v.Store(nMap) + return + } + + // make a copy + var t = reflect.TypeOf(configValue) + var nVal = reflect.New(t) + + copier.Copy(nVal.Interface(), configValue) + + val.setStruct(k, v, nVal.Interface()) + + val.v.Store(nVal.Elem().Interface()) +} + +func (val *value) setValues(ox Values, x Values) { + val.mut.Lock() + defer val.mut.Unlock() + + var configValue = val.v.Load() + + // if value is a map + // store things in a map + if val.isMap { + var mapV = configValue.(map[string]interface{}) + var nMap = make(map[string]interface{}) + + for kk, vv := range mapV { + nMap[kk] = vv + } + + for kk, vv := range x { + nMap[kk] = vv + } + + val.v.Store(nMap) + return + } + + // make a copy + var t = reflect.TypeOf(configValue) + var nVal = reflect.New(t) + + copier.Copy(nVal.Interface(), configValue) + + // reset to zero value keys not present anymore + for kk, vv := range ox { + if _, ok := x[kk]; !ok { + val.setStruct( + kk, + reflect.Zero(reflect.TypeOf(vv)).Interface(), + nVal.Interface(), + ) + } + } + + for kk, vv := range x { + val.setStruct(kk, vv, nVal.Interface()) + } + val.v.Store(nVal.Elem().Interface()) +} + +func (val *value) setStruct(k string, v interface{}, targetValue interface{}) bool { + // is a struct, find matching tag + var valTypePtr = reflect.TypeOf(targetValue) + var valType = valTypePtr.Elem() + var valValuePtr = reflect.ValueOf(targetValue) + var valValue = valValuePtr.Elem() + var set bool + + for i := 0; i < valType.NumField(); i++ { + var fieldType = valType.Field(i) + var fieldName = fieldType.Name + var tag = fieldType.Tag.Get(TagKey) + + // check tag, if it matches key + // assign v to field + if tag == k || strings.EqualFold(fieldName, k) { + var field = valValue.FieldByName(fieldType.Name) + if field.CanSet() { + field.Set(reflect.ValueOf(castValue(field.Interface(), v))) + } + set = true + continue + + // else if key has tag in prefix + } else if strings.HasPrefix(k, tag+KeySep) || + strings.HasPrefix(strings.ToLower(k), strings.ToLower(fieldName)+KeySep) { + + var nK string + + if strings.HasPrefix(k, tag+KeySep) { + nK = k[len(tag+KeySep):] + } else { + nK = k[len(fieldName+KeySep):] + } + + switch fieldType.Type.Kind() { + case reflect.Struct: + var field = valValue.FieldByName(fieldType.Name) + // if field can be set + if field.CanSet() { + var structType = field.Type() + var nVal = reflect.New(structType) + + // we copy it + copier.Copy(nVal.Interface(), field.Interface()) + + // we set the field with the new struct + if ok := val.setStruct(nK, v, nVal.Interface()); ok { + field.Set(nVal.Elem()) + set = true + } + + continue + } + case reflect.Ptr: + if fieldType.Type.Elem().Kind() == reflect.Struct { + var field = valValue.FieldByName(fieldType.Name) + if field.CanSet() { + var nVal = reflect.New(fieldType.Type.Elem()) + + // if field is not nil + // we copy it + if !field.IsNil() { + copier.Copy(nVal.Interface(), field.Interface()) + } + + if ok := val.setStruct(nK, v, nVal.Interface()); ok { + field.Set(nVal) + set = true + } + continue + } + } + } + } + } + + if !set { + val.s.cfg.Logger.Debug( + fmt.Sprintf( + "Config key %s not found in bound value", + k, + ), + ) + } + + return set +} + +func castValue(f interface{}, v interface{}) interface{} { + switch f.(type) { + case string: + return cast.ToString(v) + case bool: + return cast.ToBool(v) + case int: + return cast.ToInt(v) + case int64: + return cast.ToInt64(v) + case int32: + return cast.ToInt32(v) + case float64: + return cast.ToFloat64(v) + case float32: + return cast.ToFloat32(v) + case uint64: + return cast.ToUint64(v) + case uint32: + return cast.ToUint32(v) + case uint8: + return cast.ToUint8(v) + case []string: + return cast.ToStringSlice(v) + case []int: + return cast.ToIntSlice(v) + case time.Time: + return cast.ToTime(v) + case time.Duration: + return cast.ToDuration(v) + case map[string]interface{}: + return cast.ToStringMap(v) + case map[string]string: + return cast.ToStringMapString(v) + } + return v +} diff --git a/value_test.go b/value_test.go new file mode 100644 index 00000000..e9fab8c5 --- /dev/null +++ b/value_test.go @@ -0,0 +1,211 @@ +package konfig + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBind(t *testing.T) { + t.Run( + "panic invalid type", + func(t *testing.T) { + var s = Instance() + require.Panics(t, func() { s.Bind(1) }) + }, + ) + + t.Run( + "valid type map", + func(t *testing.T) { + var s = Instance() + var m = make(map[string]interface{}) + require.NotPanics(t, func() { s.Bind(m) }) + }, + ) + + t.Run( + "valid type struct", + func(t *testing.T) { + type testConfig struct { + v string `konfig:"v"` + } + var s = Instance() + var tc testConfig + require.NotPanics(t, func() { s.Bind(tc) }) + }, + ) + +} + +func TestSetStruct(t *testing.T) { + + t.Run( + "valid type struct", + func(t *testing.T) { + type TestConfigSub struct { + VV string `konfig:"vv"` + TT int `konfig:"tt"` + B bool `konfig:"bool"` + F float64 `konfig:"float64"` + U uint64 `konfig:"uint64"` + I64 int64 `konfig:"int64"` + } + type TestConfig struct { + V string `konfig:"v"` + T TestConfigSub `konfig:"sub"` + SubT *TestConfigSub + } + + var expectedConfig = TestConfig{ + V: "test", + T: TestConfigSub{ + VV: "test2", + TT: 1, + B: true, + F: 1.9, + I64: 1, + }, + SubT: &TestConfigSub{ + VV: "", + TT: 2, + }, + } + + Init(DefaultConfig()) + + var tc TestConfig + require.NotPanics(t, func() { Bind(tc) }) + + var v = Values{ + "v": "test", + "sub.vv": "test2", + "sub.tt": 1, + "subt.tt": 2, + "sub.bool": true, + "sub.float64": 1.9, + "sub.int64": int64(1), + } + + v.load(Values{ + "v": "a", + }, instance()) + + var configValue = Value().(TestConfig) + require.Equal(t, "test", configValue.V) + require.Equal(t, "test2", configValue.T.VV) + require.Equal(t, 1, configValue.T.TT) + require.Equal(t, 2, configValue.SubT.TT) + require.Equal(t, true, configValue.T.B) + require.Equal(t, 1.9, configValue.T.F) + require.Equal(t, int64(1), configValue.T.I64) + + require.Equal(t, expectedConfig, Value()) + + var vv = Values{ + "v": "test", + "sub.vv": "test2", + } + + vv.load(v, instance()) + + configValue = Value().(TestConfig) + require.Equal(t, "test", configValue.V) + require.Equal(t, "test2", configValue.T.VV) + require.Equal(t, 0, configValue.T.TT) + require.Equal(t, 0, configValue.SubT.TT) + }, + ) + + t.Run( + "valid type struct", + func(t *testing.T) { + type TestConfigSub struct { + VV string `konfig:"vv"` + TT int `konfig:"tt"` + } + type TestConfig struct { + V string `konfig:"v"` + T TestConfigSub `konfig:"sub"` + SubT *TestConfigSub + } + + var expectedConfig = TestConfig{ + V: "test", + T: TestConfigSub{ + VV: "bar", + TT: 1, + }, + SubT: &TestConfigSub{ + VV: "", + TT: 2, + }, + } + + Init(DefaultConfig()) + + var tc TestConfig + require.NotPanics(t, func() { Bind(tc) }) + + Set("v", "test") + Set("sub.vv", "bar") + Set("sub.tt", 1) + Set("subt.tt", 2) + + var configValue = Value().(TestConfig) + require.Equal(t, "test", configValue.V) + require.Equal(t, "bar", configValue.T.VV) + require.Equal(t, 1, configValue.T.TT) + require.Equal(t, 2, configValue.SubT.TT) + + require.Equal(t, expectedConfig, Value()) + + }, + ) + + t.Run( + "valid type map", + func(t *testing.T) { + Init(DefaultConfig()) + + var tc = make(map[string]interface{}) + require.NotPanics(t, func() { Bind(tc) }) + + Set("v", "test") + Set("sub.vv", "bar") + Set("sub.tt", 1) + Set("subt.tt", 2) + + var configValue = Value().(map[string]interface{}) + require.Equal(t, "test", configValue["v"]) + require.Equal(t, "bar", configValue["sub.vv"]) + require.Equal(t, 1, configValue["sub.tt"]) + require.Equal(t, 2, configValue["subt.tt"]) + }, + ) + + t.Run( + "valid type map", + func(t *testing.T) { + Init(DefaultConfig()) + + var tc = make(map[string]interface{}) + require.NotPanics(t, func() { Bind(tc) }) + + var v = Values{ + "v": "test", + "sub.vv": "test2", + "sub.tt": 1, + "subt.tt": 2, + } + + v.load(Values{}, instance()) + + var configValue = Value().(map[string]interface{}) + require.Equal(t, "test", configValue["v"]) + require.Equal(t, "test2", configValue["sub.vv"]) + require.Equal(t, 1, configValue["sub.tt"]) + require.Equal(t, 2, configValue["subt.tt"]) + }, + ) +} diff --git a/watcher.go b/watcher.go new file mode 100644 index 00000000..72e4551c --- /dev/null +++ b/watcher.go @@ -0,0 +1,65 @@ +package konfig + +// Watcher is the interface implementing a config watcher. +// Config watcher trigger loaders. A file watcher or a simple +// Timer can be valid watchers. +type Watcher interface { + // Start starts the watcher, it must no be blocking. + Start() error + // Done indicate wether the watcher is done or not + Done() <-chan struct{} + // Watch should block until an event unlocks it + Watch() <-chan struct{} + // Close closes the watcher, it returns a non nil error if it is already closed + // or something prevents it from closing properly. + Close() error + // Err returns telling why the watcher closed + Err() error +} + +// NopWatcher is a nil watcher +type NopWatcher struct { + Watcher +} + +// Done returns an already closed channel +func (NopWatcher) Done() <-chan struct{} { + var c = make(chan struct{}) + close(c) + return c +} + +// Watch implements a basic watch that waits forever +func (NopWatcher) Watch() <-chan struct{} { + var c = make(chan struct{}) + return c +} + +// Start implements watcher interface and always returns a nil error +func (NopWatcher) Start() error { + return nil +} + +// Watch starts the watchers on loaders +func Watch() error { + return instance().Watch() +} + +// Watch starts the watchers on loaders +func (c *store) Watch() error { + + // if metrics are enabled, we register them in prometheus + if c.cfg.Metrics { + if err := c.registerMetrics(); err != nil { + return err + } + } + + for _, wl := range c.WatcherLoaders { + if err := wl.Start(); err != nil { + return err + } + go c.watchLoader(wl) + } + return nil +} diff --git a/watcher/kwfile/filewatcher.go b/watcher/kwfile/filewatcher.go new file mode 100644 index 00000000..c7075fa3 --- /dev/null +++ b/watcher/kwfile/filewatcher.go @@ -0,0 +1,122 @@ +package kwfile + +import ( + "fmt" + "os" + "time" + + "github.com/lalamove/konfig" + "github.com/lalamove/nui/nlogger" + "github.com/radovskyb/watcher" +) + +var _ konfig.Watcher = (*FileWatcher)(nil) +var defaultRate = 10 * time.Second + +// Config is the config of a FileWatcher +type Config struct { + // Files is the path to the files to watch + Files []string + // Rate is the rate at wich the file is watched + Rate time.Duration + // Debug sets the debug mode on the filewatcher + Debug bool + // Logger is the logger used to print messages + Logger nlogger.Logger +} + +// FileWatcher watches over a file given in the config +type FileWatcher struct { + cfg *Config + w *watcher.Watcher + err error + watchChan chan struct{} +} + +// New creates a new FileWatcher from the given *Config cfg +func New(cfg *Config) *FileWatcher { + if cfg.Logger == nil { + cfg.Logger = defaultLogger() + } + if cfg.Rate == 0 { + cfg.Rate = defaultRate + } + + var w = watcher.New() + + for _, file := range cfg.Files { + cfg.Logger.Info("adding file to watch: " + file) + if err := w.Add(file); err != nil { + panic(err) + } + } + + return &FileWatcher{ + cfg: cfg, + w: w, + watchChan: make(chan struct{}), + } +} + +// Done indicates wether the filewatcher is done +func (fw *FileWatcher) Done() <-chan struct{} { + return fw.w.Closed +} + +// Start starts the file watcher +func (fw *FileWatcher) Start() error { + go fw.watch() + go func() error { + if err := fw.w.Start(fw.cfg.Rate); err != nil { + return err + } + return nil + }() + return nil +} + +// Watch return the channel to which events are written +func (fw *FileWatcher) Watch() <-chan struct{} { + return fw.watchChan +} + +func (fw *FileWatcher) watch() { + for { + select { + // we get an event, write to the struct chan + // log if debug mode + case e := <-fw.w.Event: + if fw.cfg.Debug { + fw.cfg.Logger.Debug(fmt.Sprintf( + "Event received %v", + e, + )) + } + fw.watchChan <- struct{}{} + case err := <-fw.w.Error: + // log error + fw.cfg.Logger.Error(err.Error()) + fw.err = err + fw.Close() + return + case <-fw.w.Closed: + // watcher is closed, return + return + } + } +} + +// Close closes the FileWatcher +func (fw *FileWatcher) Close() error { + fw.w.Close() + return nil +} + +// Err returns the file watcher error +func (fw *FileWatcher) Err() error { + return fw.err +} + +func defaultLogger() nlogger.Logger { + return nlogger.New(os.Stdout, "FILWATCHER | ") +} diff --git a/watcher/kwpoll/pollwatcher.go b/watcher/kwpoll/pollwatcher.go new file mode 100644 index 00000000..c9d7821c --- /dev/null +++ b/watcher/kwpoll/pollwatcher.go @@ -0,0 +1,186 @@ +package kwpoll + +import ( + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/go-test/deep" + "github.com/lalamove/konfig" + "github.com/lalamove/nui/nlogger" +) + +var ( + _ konfig.Watcher = (*PollWatcher)(nil) + // ErrNoGetter is the error returned when no getter is set and Diff is set to true + ErrNoGetter = errors.New("You must give a non nil getter to the poll diff watcher") + // ErrAlreadyClosed is the error returned when trying to close an already closed PollDiffWatcher + ErrAlreadyClosed = errors.New("PollDiffWatcher already closed") +) + +// Rater is an interface that exposes a single +// Time method which returns the time until the next tick +type Rater interface { + Time() time.Duration +} + +// Getter is the interface to implement to fetch data to compare +type Getter interface { + Get() (interface{}, error) +} + +// Time is a time.Duration which implements the Rater interface +type Time time.Duration + +// Time returns the time.Duration +func (t Time) Time() time.Duration { + return time.Duration(t) +} + +// Config is the config of a PollWatcher +type Config struct { + // Rater is the rater the PollWatcher calls to get the duration until the next tick + Rater Rater + // Debug sets the debug mode + Debug bool + // Logger is the logger used to log debug messages + Logger nlogger.Logger + // Diff tells wether we should check for diffs + // If diff is set, a Getter is required + Diff bool + // Getter is a getter to fetch data to check diff + Getter Getter + // InitValue is the initial value to compare with whe Diff is true + InitValue interface{} +} + +// PollWatcher is a konfig.Watcher that sends events every x time given in the konfig. +type PollWatcher struct { + cfg *Config + err error + pv interface{} + watchChan chan struct{} + done chan struct{} +} + +// New creates a new PollWatcher fromt the given config +func New(cfg *Config) *PollWatcher { + if cfg.Diff && cfg.Getter == nil { + panic(ErrNoGetter) + } + if cfg.Logger == nil { + cfg.Logger = defaultLogger() + } + + return &PollWatcher{ + cfg: cfg, + pv: cfg.InitValue, + done: make(chan struct{}), + watchChan: make(chan struct{}), + } +} + +// Done indicates wether the watcher is done or not +func (t *PollWatcher) Done() <-chan struct{} { + return t.done +} + +// Start starts the ticker watcher +func (t *PollWatcher) Start() error { + if t.cfg.Debug { + t.cfg.Logger.Debug( + fmt.Sprintf( + "Starting ticker watcher with rate: %d", + t.cfg.Rater.Time()/time.Second, + ), + ) + } + go t.watch() + return nil +} + +// Watch returns the channel to which events are written +func (t *PollWatcher) Watch() <-chan struct{} { + return t.watchChan +} + +// Err returns the poll watcher error +func (t *PollWatcher) Err() error { + return t.err +} + +func (t *PollWatcher) watch() { + var rate = t.cfg.Rater.Time() + + t.cfg.Logger.Debug( + fmt.Sprintf( + "Waiting rater duration: %v seconds", + rate/time.Second, + ), + ) + + time.Sleep(rate) + for { + select { + case <-t.done: + default: + if t.cfg.Debug { + t.cfg.Logger.Debug("Tick") + } + if t.cfg.Diff { + + t.cfg.Logger.Debug( + "Checking difference", + ) + + var r, err = t.cfg.Getter.Get() + // We got error, we close + if err != nil { + t.cfg.Logger.Error(err.Error()) + t.err = err + t.Close() + return + } + + if diff := deep.Equal(t.pv, r); diff != nil { + if t.cfg.Debug { + t.cfg.Logger.Debug( + "Value is different: " + strings.Join(diff, "\n"), + ) + } + t.watchChan <- struct{}{} + t.pv = r + } + + t.cfg.Logger.Debug( + "Value are the same, not updating", + ) + } else { + + t.cfg.Logger.Debug( + "Sending watch event", + ) + + t.watchChan <- struct{}{} + } + time.Sleep(t.cfg.Rater.Time()) + } + } +} + +// Close closes the PollWatcher +func (t *PollWatcher) Close() error { + select { + case <-t.done: + return ErrAlreadyClosed + default: + close(t.done) + } + return nil +} + +func defaultLogger() nlogger.Logger { + return nlogger.New(os.Stdout, "POLLWATCHER | ") +} diff --git a/watcher/kwpoll/pollwatcher_test.go b/watcher/kwpoll/pollwatcher_test.go new file mode 100644 index 00000000..80a4dca4 --- /dev/null +++ b/watcher/kwpoll/pollwatcher_test.go @@ -0,0 +1,66 @@ +package kwpoll + +import ( + "testing" + "time" + + gomock "github.com/golang/mock/gomock" + "github.com/lalamove/konfig/mocks" +) + +func TestWatcher(t *testing.T) { + t.Run( + "basic watcher, no diff", + func(t *testing.T) { + var w = New(&Config{ + Rater: Time(100 * time.Millisecond), + Debug: true, + }) + w.Start() + time.Sleep(200 * time.Millisecond) + var timer = time.NewTimer(100 * time.Millisecond) + select { + case <-timer.C: + t.Error("watcher should have ticked") + w.Close() + case <-w.Watch(): + w.Close() + return + } + }, + ) + t.Run( + "watcher diff", + func(t *testing.T) { + var ctrl = gomock.NewController(t) + defer ctrl.Finish() + + var g = mocks.NewMockGetter(ctrl) + + gomock.InOrder( + g.EXPECT().Get().Times(1).Return(nil, nil), + g.EXPECT().Get().Times(1).Return(1, nil), + ) + + var w = New(&Config{ + Rater: Time(100 * time.Millisecond), + Getter: g, + Diff: true, + Debug: true, + }) + w.Start() + + time.Sleep(200 * time.Millisecond) + var timer = time.NewTimer(200 * time.Millisecond) + select { + case <-timer.C: + t.Error("watcher should have ticked") + w.Close() + case <-w.Watch(): + w.Close() + return + } + + }, + ) +} diff --git a/watcher_mock_test.go b/watcher_mock_test.go new file mode 100644 index 00000000..a5469022 --- /dev/null +++ b/watcher_mock_test.go @@ -0,0 +1,103 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./watcher.go + +// Package konfig is a generated GoMock package. +package konfig + +import ( + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockWatcher is a mock of Watcher interface +type MockWatcher struct { + ctrl *gomock.Controller + recorder *MockWatcherMockRecorder +} + +// MockWatcherMockRecorder is the mock recorder for MockWatcher +type MockWatcherMockRecorder struct { + mock *MockWatcher +} + +// NewMockWatcher creates a new mock instance +func NewMockWatcher(ctrl *gomock.Controller) *MockWatcher { + mock := &MockWatcher{ctrl: ctrl} + mock.recorder = &MockWatcherMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockWatcher) EXPECT() *MockWatcherMockRecorder { + return m.recorder +} + +// Start mocks base method +func (m *MockWatcher) Start() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Start") + ret0, _ := ret[0].(error) + return ret0 +} + +// Start indicates an expected call of Start +func (mr *MockWatcherMockRecorder) Start() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockWatcher)(nil).Start)) +} + +// Done mocks base method +func (m *MockWatcher) Done() <-chan struct{} { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Done") + ret0, _ := ret[0].(<-chan struct{}) + return ret0 +} + +// Done indicates an expected call of Done +func (mr *MockWatcherMockRecorder) Done() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Done", reflect.TypeOf((*MockWatcher)(nil).Done)) +} + +// Watch mocks base method +func (m *MockWatcher) Watch() <-chan struct{} { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Watch") + ret0, _ := ret[0].(<-chan struct{}) + return ret0 +} + +// Watch indicates an expected call of Watch +func (mr *MockWatcherMockRecorder) Watch() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Watch", reflect.TypeOf((*MockWatcher)(nil).Watch)) +} + +// Close mocks base method +func (m *MockWatcher) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close +func (mr *MockWatcherMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockWatcher)(nil).Close)) +} + +// Err mocks base method +func (m *MockWatcher) Err() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Err") + ret0, _ := ret[0].(error) + return ret0 +} + +// Err indicates an expected call of Err +func (mr *MockWatcherMockRecorder) Err() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Err", reflect.TypeOf((*MockWatcher)(nil).Err)) +}