Skip to content
This repository has been archived by the owner on Apr 27, 2022. It is now read-only.

Commit

Permalink
fix etcd list recurcive
Browse files Browse the repository at this point in the history
  • Loading branch information
Martin committed Jun 23, 2016
1 parent 5657817 commit ddc5338
Show file tree
Hide file tree
Showing 4 changed files with 223 additions and 44 deletions.
3 changes: 2 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
.vscode
debug.test
coverage.out
example*
kv_local_test.go
scripts
53 changes: 47 additions & 6 deletions kv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,14 +39,16 @@ func (kv *KvSource) Parse(cmd *flaeg.Command) (*flaeg.Command, error) {

// LoadConfig loads data from the KV Store into the config structure (given by reference)
func (kv *KvSource) LoadConfig(config interface{}) error {
pairs, err := kv.List(kv.Prefix)
if err != nil {
pairs := map[string][]byte{}
if err := kv.ListRecursive(kv.Prefix, pairs); err != nil {
return err
}
mapstruct, err := generateMapstructure(pairs, kv.Prefix)
// fmt.Printf("pairs : %#v\n", pairs)
mapstruct, err := generateMapstructure(convertPairs(pairs), kv.Prefix)
if err != nil {
return err
}
// fmt.Printf("mapstruct : %#v\n", mapstruct)
configDecoder := &mapstructure.DecoderConfig{
Metadata: nil,
Result: config,
Expand All @@ -67,8 +69,8 @@ func generateMapstructure(pairs []*store.KVPair, prefix string) (map[string]inte
raw := make(map[string]interface{})
for _, p := range pairs {
// Trim the prefix off our key first
key := strings.TrimPrefix(p.Key, prefix+"/")
raw, err := processKV(key, string(p.Value), raw)
key := strings.TrimPrefix(strings.Trim(p.Key, "/"), strings.Trim(prefix, "/")+"/")
raw, err := processKV(key, p.Value, raw)
if err != nil {
return raw, err
}
Expand All @@ -77,7 +79,7 @@ func generateMapstructure(pairs []*store.KVPair, prefix string) (map[string]inte
return raw, nil
}

func processKV(key string, v string, raw map[string]interface{}) (map[string]interface{}, error) {
func processKV(key string, v []byte, raw map[string]interface{}) (map[string]interface{}, error) {
// Determine which map we're writing the value to. We split by '/'
// to determine any sub-maps that need to be created.
m := raw
Expand Down Expand Up @@ -226,3 +228,42 @@ func collateKvRecursive(objValue reflect.Value, kv map[string]string, key string
}
return nil
}

// ListRecursive lists all key value childrens under key
func (kv *KvSource) ListRecursive(key string, pairs map[string][]byte) error {
pairsN1, err := kv.List(key)
if err != nil {
return err
}
if len(pairsN1) == 0 {
pairLeaf, err := kv.Get(key)
if err != nil {
return err
}
if pairLeaf == nil {
return nil
}
pairs[pairLeaf.Key] = pairLeaf.Value
return nil
}
for _, p := range pairsN1 {
err := kv.ListRecursive(p.Key, pairs)
if err != nil {
return err
}
}
return nil
}

func convertPairs(pairs map[string][]byte) []*store.KVPair {
slicePairs := make([]*store.KVPair, len(pairs))
i := 0
for k, v := range pairs {
slicePairs[i] = &store.KVPair{
Key: k,
Value: v,
}
i++
}
return slicePairs
}
204 changes: 174 additions & 30 deletions kv_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -310,8 +310,15 @@ func (s *Mock) List(prefix string) ([]*store.KVPair, error) {
}
kv := []*store.KVPair{}
for _, kvPair := range s.KVPairs {
if strings.HasPrefix(kvPair.Key, prefix) {
kv = append(kv, kvPair)
if strings.HasPrefix(kvPair.Key, prefix+"/") {
if secondSlashIndex := strings.IndexRune(kvPair.Key[len(prefix)+1:], '/'); secondSlashIndex == -1 {
kv = append(kv, kvPair)
} else {
dir := &store.KVPair{
Key: kvPair.Key[:secondSlashIndex+len(prefix)+1],
}
kv = append(kv, dir)
}
}
}
return kv, nil
Expand Down Expand Up @@ -689,35 +696,57 @@ func TestParseKvSourceNestedPtrsNil(t *testing.T) {
t.Fatalf("\nexpected\t: %s\ngot\t\t\t: %s\n", printCheck, printResult)
}
}
func TestIntegrationMockList(t *testing.T) {
kv := &Mock{
KVPairs: []*store.KVPair{
{
Key: "test/ptrstruct1/s1int",
Value: []byte("28"),
},
{
Key: "test/durationfield",
Value: []byte("28"),

func TestParseKvSourceMap(t *testing.T) {
//Init
config := &struct {
Vmap map[string]int
}{}

//Test
rootCmd := &flaeg.Command{
Name: "test",
Description: "description test",
Config: config,
DefaultPointersConfig: config,
Run: func() error { return nil },
}
kv := &KvSource{
&Mock{
KVPairs: []*store.KVPair{
{
Key: "prefix/vmap/toto",
Value: []byte("1"),
},
{
Key: "prefix/vmap/tata",
Value: []byte("2"),
},
{
Key: "prefix/vmap/titi",
Value: []byte("3"),
},
},
},
"prefix",
}
pairs, err := kv.List("test/")
if err != nil {
t.Fatalf("Error : %s", err)
}
//check
if len(pairs) != 2 {
t.Fatalf("Expected 2 pairs got %d", len(pairs))
if _, err := kv.Parse(rootCmd); err != nil {
t.Fatalf("Error %s", err)
}
check := map[string][]byte{
"test/ptrstruct1/s1int": []byte("28"),
"test/durationfield": []byte("28"),

//Check
check := &struct {
Vmap map[string]int
}{
Vmap: map[string]int{
"toto": 1,
"tata": 2,
"titi": 3,
},
}
for _, p := range pairs {
if !reflect.DeepEqual(p.Value, check[p.Key]) {
t.Fatalf("key %s expected value %s got %s", p.Key, check[p.Key], p.Value)
}

if !reflect.DeepEqual(check, rootCmd.Config) {
t.Fatalf("\nexpected\t: %#v\ngot\t\t\t: %#v\n", check, rootCmd.Config)
}
}

Expand Down Expand Up @@ -1068,16 +1097,17 @@ func TestStoreConfigEmbeddedSquash(t *testing.T) {
"prefix/bar2": "titi",
"prefix/vfoo": "toto",
}
result, err := kv.List("")
result := map[string][]byte{}
err := kv.ListRecursive("prefix", result)
if err != nil {
t.Fatalf("Error : %s", err)
}
if len(result) != len(checkMap) {
t.Fatalf("length of kv.List is not %d", len(checkMap))
}
for _, pair := range result {
if string(pair.Value) != checkMap[pair.Key] {
t.Fatalf("Key %s\nExpected value %s, got %s", pair.Key, pair.Value, checkMap[pair.Key])
for k, v := range result {
if string(v) != checkMap[k] {
t.Fatalf("Key %s\nExpected value %s, got %s", k, v, checkMap[k])
}

}
Expand Down Expand Up @@ -1136,3 +1166,117 @@ func TestCollateKvPairsShortNameUnexported(t *testing.T) {
t.Fatalf("Expected %s\nGot %s", check, kv)
}
}
func TestListRecursive5Levels(t *testing.T) {
kv := &KvSource{
&Mock{
KVPairs: []*store.KVPair{
{
Key: "prefix/l1",
Value: []byte("level1"),
},
{
Key: "prefix/d1/l1",
Value: []byte("level2"),
},
{
Key: "prefix/d1/l2",
Value: []byte("level2"),
},
{
Key: "prefix/d2/d1/l1",
Value: []byte("level3"),
},
{
Key: "prefix/d3/d2/d1/d1/d1",
Value: []byte("level5"),
},
},
},
"prefix",
}
pairs := map[string][]byte{}
err := kv.ListRecursive(kv.Prefix, pairs)
if err != nil {
t.Fatalf("Error : %s", err)
}

//check
check := map[string][]byte{
"prefix/l1": []byte("level1"),
"prefix/d1/l1": []byte("level2"),
"prefix/d1/l2": []byte("level2"),
"prefix/d2/d1/l1": []byte("level3"),
"prefix/d3/d2/d1/d1/d1": []byte("level5"),
}
if len(pairs) != len(check) {
t.Fatalf("Expected length %d, got %d", len(check), len(pairs))
}
for k, v := range pairs {
if !reflect.DeepEqual(v, check[k]) {
t.Fatalf("Key %s\nExpected %s\nGot %s", k, check[k], v)
}
}
}

func TestListRecursiveEmpty(t *testing.T) {
kv := &KvSource{
&Mock{
KVPairs: []*store.KVPair{},
},
"prefix",
}
pairs := map[string][]byte{}
err := kv.ListRecursive(kv.Prefix, pairs)
if err != nil {
t.Fatalf("Error : %s", err)
}

//check
check := map[string][]byte{}
if len(pairs) != len(check) {
t.Fatalf("Expected length %d, got %d", len(check), len(pairs))
}
}

func TestConvertPairs5Levels(t *testing.T) {
input := map[string][]byte{
"prefix/l1": []byte("level1"),
"prefix/d1/l1": []byte("level2"),
"prefix/d1/l2": []byte("level2"),
"prefix/d2/d1/l1": []byte("level3"),
"prefix/d3/d2/d1/d1/d1": []byte("level5"),
}
//test
output := convertPairs(input)

//check
check := []*store.KVPair{
{
Key: "prefix/l1",
Value: []byte("level1"),
},
{
Key: "prefix/d1/l1",
Value: []byte("level2"),
},
{
Key: "prefix/d1/l2",
Value: []byte("level2"),
},
{
Key: "prefix/d2/d1/l1",
Value: []byte("level3"),
},
{
Key: "prefix/d3/d2/d1/d1/d1",
Value: []byte("level5"),
},
}

if len(output) != len(check) {
t.Fatalf("Expected length %d, got %d", len(check), len(output))
}
if !reflect.DeepEqual(output, check) {
t.Fatalf("Expected %#v\nGot %#v", check, output)
}
}
7 changes: 0 additions & 7 deletions scripts/consul-init.sh
View file

This file was deleted.

0 comments on commit ddc5338

Please sign in to comment.