From 70bdc500d7b556184a0105e231446edbcef64d10 Mon Sep 17 00:00:00 2001 From: Vladislav Mitov Date: Mon, 10 Apr 2017 00:26:14 +0300 Subject: [PATCH] Add ignore keys functionality to the Consul storage --- README.md | 1 + cmd/consulstorage.go | 27 +++++++++++++++++++++------ cmd/consulstorage_test.go | 31 +++++++++++++++++++++++++------ 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index ede3318..95778a9 100644 --- a/README.md +++ b/README.md @@ -37,5 +37,6 @@ casper build * **storage** - Storage is the system that Casper menages. Storage will have two keys - `type` and `config` where `config` contains all the configurations for the storage specified by `type`. Currently there are 2 available: * consul - Consul (Formats: json, yaml) * addr - address of the consul instance e.g: `localhost:8500` + * ignore - keys given the value of this setting in configuration will be ignored by Casper. The default such value is "_ignore" * file - File (Formats: string) * path - path to the file diff --git a/cmd/consulstorage.go b/cmd/consulstorage.go index 9722d7f..ec01fa0 100644 --- a/cmd/consulstorage.go +++ b/cmd/consulstorage.go @@ -13,6 +13,10 @@ import ( "github.com/miracl/casper/lib/diff" ) +const ( + defaultIgnoreVal = "_ignore" +) + // ConsulKV is interface that consul KV type implements. // Defined and used mainly for testing. type ConsulKV interface { @@ -26,7 +30,8 @@ var consulFormats = []string{"json", "yaml", "jsonraw"} type consulStorage struct { kv ConsulKV - formats []string + formats []string + ignoreVal string } var errConsulAddr = errors.New("Consul addr is invalid type") @@ -45,10 +50,15 @@ func newConsulStorageConfig(config map[string]interface{}) (storage, error) { return nil, errConsulAddr } - return newConsulStorage(strAddr) + ignoreVal, ok := config["ignore"].(string) + if !ok || ignoreVal == "" { + ignoreVal = defaultIgnoreVal + } + + return newConsulStorage(strAddr, ignoreVal) } -func newConsulStorage(addr string) (storage, error) { +func newConsulStorage(addr, ignoreVal string) (storage, error) { cfg := &api.Config{} if addr != "" { addr, err := url.Parse(addr) @@ -64,7 +74,7 @@ func newConsulStorage(addr string) (storage, error) { return nil, err } - return &consulStorage{client.KV(), consulFormats}, nil + return &consulStorage{client.KV(), consulFormats, ignoreVal}, nil } func (s consulStorage) String(format string) (string, error) { @@ -94,7 +104,7 @@ func (s consulStorage) GetChanges(config []byte, format, key string) (changes, e return nil, err } - return getChanges(pairs, config, format, key) + return getChanges(pairs, config, format, key, s.ignoreVal) } func (consulStorage) Diff(cs changes, pretty bool) string { @@ -143,7 +153,7 @@ func kvPairsToString(pairs api.KVPairs, format string) string { return string(res) } -func getChanges(pairs api.KVPairs, config []byte, format, key string) (changes, error) { +func getChanges(pairs api.KVPairs, config []byte, format, key, ignoreVal string) (changes, error) { consulChanges, err := consul.GetChanges(pairs, config, format) if err != nil { return nil, err @@ -151,6 +161,11 @@ func getChanges(pairs api.KVPairs, config []byte, format, key string) (changes, kvChanges := diff.KVChanges{} for _, c := range consulChanges { + // Skip ignored pairs + if ignoreVal != "" && c.NewVal == ignoreVal { + continue + } + if key != "" && key != c.Key { continue } diff --git a/cmd/consulstorage_test.go b/cmd/consulstorage_test.go index d17efa8..917fbe9 100644 --- a/cmd/consulstorage_test.go +++ b/cmd/consulstorage_test.go @@ -4,7 +4,8 @@ import ( "bytes" "errors" "fmt" - "reflect" + "sort" + "strings" "testing" "github.com/hashicorp/consul/api" @@ -86,7 +87,7 @@ func TestConsulStorageString(t *testing.T) { for i, tc := range testCases { t.Run(fmt.Sprintf("Case%v", i), func(t *testing.T) { - s := &consulStorage{&ConsulKVMock{list: tc.list, listErr: tc.listErr}, []string{"jsonraw"}} + s := &consulStorage{&ConsulKVMock{list: tc.list, listErr: tc.listErr}, []string{"jsonraw"}, defaultIgnoreVal} str, err := s.String("jsonraw") if err != tc.err { t.Fatalf("Got %v; want %v", err, tc.err) @@ -125,11 +126,28 @@ func TestConsulStoragePush(t *testing.T) { }, []string{"key2"}, }, + { + api.KVPairs{ + &api.KVPair{Key: "key1", Value: []byte("val1")}, + &api.KVPair{Key: "key2", Value: []byte("val2")}, + &api.KVPair{Key: "key3", Value: []byte("val")}, + }, + `{"key1":"val1","key2":"_ignore","key3":"val3","key4":"val4"}`, + "" + + "-key3=val\n" + + "+key3=val3\n" + + "+key4=val4\n", + api.KVPairs{ + &api.KVPair{Key: "key3", Value: []byte("val3")}, + &api.KVPair{Key: "key4", Value: []byte("val4")}, + }, + []string{}, + }, } for i, tc := range testCases { t.Run(fmt.Sprintf("Case%v", i), func(t *testing.T) { - s := &consulStorage{&ConsulKVMock{list: tc.list}, []string{"jsonraw "}} + s := &consulStorage{&ConsulKVMock{list: tc.list}, []string{"jsonraw "}, defaultIgnoreVal} cs, err := s.GetChanges([]byte(tc.config), "jsonraw", "") if err != nil { @@ -147,7 +165,8 @@ func TestConsulStoragePush(t *testing.T) { } kv := s.kv.(*ConsulKVMock) - if !reflect.DeepEqual(kv.dels, tc.dels) { + sort.Strings(kv.dels) + if strings.Join(kv.dels, ",") != strings.Join(tc.dels, ",") { t.Errorf("Got `%v`; want `%v`", kv.dels, tc.dels) } @@ -200,7 +219,7 @@ func TestConsulStorageFormats(t *testing.T) { for i, tc := range testCases { t.Run(fmt.Sprintf("Case%v", i), func(t *testing.T) { - s := &consulStorage{&ConsulKVMock{}, tc.formats} + s := &consulStorage{&ConsulKVMock{}, tc.formats, defaultIgnoreVal} if s.FormatIsValid(tc.fmt) != tc.valid { t.Errorf("%v should have been valid:%v", tc.fmt, tc.valid) @@ -342,7 +361,7 @@ func TestGetChanges(t *testing.T) { for i, tc := range testCases { t.Run(fmt.Sprintf("Case%v", i), func(t *testing.T) { - ch, err := getChanges(tc.pairs, []byte(tc.config), tc.format, tc.key) + ch, err := getChanges(tc.pairs, []byte(tc.config), tc.format, tc.key, defaultIgnoreVal) if tc.ok != (err == nil) { if err != nil {