folder list

TODO.md

@@ -2,9 +2,10 @@
 
 ## Soon
 
-- CLI tests
-- path functions
+- api folder funtions
+- cli path/folder functions
 - api context and cancellations?
+- add sort as an option in cli and api
 
 ## Upcoming
 
@@ -13,7 +14,6 @@
 - API example in readme
 - doc.go in api
 - further parallelize tests?
-- cli option to sort output
 - graceful worker shutdown <https://callistaenterprise.se/blogg/teknik/2019/10/05/go-worker-cancellation/>
 - cli checks for updates <https://github.com/tcnksm/go-latest>
 - CI to make sure api Version() stays up to date with tags

go.mod

@@ -14,9 +14,9 @@ require (
 	github.com/spf13/cobra v0.0.7
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.5.1
-	golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect
+	golang.org/x/crypto v0.0.0-20200403201458-baeed622b8d8 // indirect
 	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
-	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a // indirect
+	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
 	gopkg.in/square/go-jose.v2 v2.4.1 // indirect
 )

go.sum

@@ -661,6 +661,8 @@ golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM=
 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200403201458-baeed622b8d8 h1:fpnn/HnJONpIu6hkXi1u/7rR0NzilgWr4T0JmWkEitk=
+golang.org/x/crypto v0.0.0-20200403201458-baeed622b8d8/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

vaku/client.go

@@ -22,8 +22,8 @@ type Client struct {
 	srcL logical
 	dstL logical
 
-	// workers is the max number of concurrent operations we'll run.
-	workers uint
+	// workers is the max number of concurrent operations against vault.
+	workers int
 
 	// absolutepath if the absolution path is desired instead of the relative path.
 	absolutepath bool
@@ -71,18 +71,18 @@ func (o withDstVaultClient) apply(c *Client) error {
 	return nil
 }
 
-// WithWorkers sets the maximum number of goroutines that will be used to run folder based
-// operations. Default value is 10. A stable and well-operated Vault server should be able to handle
-// 100 or more without issue. Use with caution and tune specifically to your environment and storage
-// backend.
-func WithWorkers(n uint) Option {
+// WithWorkers sets the maximum number of goroutines that access Vault at any given time. Does not
+// cap the number of goroutines overall. Default value is 10. A stable and well-operated Vault
+// server should be able to handle 100 or more without issue. Use with caution and tune specifically
+// to your environment and storage backend.
+func WithWorkers(n int) Option {
 	return withWorkers(n)
 }
 
 type withWorkers uint
 
 func (o withWorkers) apply(c *Client) error {
-	c.workers = uint(o)
+	c.workers = int(o)
 	return nil
 }
 

vaku/folder_list.go

@@ -0,0 +1,118 @@
+package vaku
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	// ErrFolderList when FolderList fails.
+	ErrFolderList = errors.New("folder list")
+)
+
+// FolderList recursively lists the provided path and all subpaths.
+func (c *Client) FolderList(ctx context.Context, p string) ([]string, error) {
+	resC, errC := c.FolderListChan(ctx, p)
+
+	// read results and errors. send on errC signifies done (can be nil).
+	var out []string
+	for {
+		select {
+		case res := <-resC:
+			out = append(out, res)
+		case err := <-errC:
+			if err != nil {
+				return nil, err
+			}
+			return out, nil
+		}
+	}
+}
+
+// FolderListChan recursively lists the provided path and all subpaths. Returns an unbuffered
+// channel that can be read until close and an error channel that sends either the first error or
+// nil when the work is done.
+func (c *Client) FolderListChan(ctx context.Context, p string) (<-chan string, <-chan error) {
+	// input must be a folder (end in "/")
+	root := MakeFolder(p)
+
+	// eg manages workers reading from the paths channel
+	eg, ctx := errgroup.WithContext(ctx)
+	// wg tracks when to close the paths channel
+	var wg sync.WaitGroup
+
+	// pathC is paths to be processed
+	pathC := make(chan string)
+	// resC is processed paths
+	resC := make(chan string)
+
+	// add root path to paths
+	wg.Add(1)
+	go func(p string) { pathC <- p }(root)
+
+	// fan out and process paths
+	for i := 0; i < c.workers; i++ {
+		eg.Go(func() error { return c.folderListWork(ctx, root, &wg, pathC, resC) })
+	}
+
+	// close pathC once all have been processed or when the group is cancelled
+	eg.Go(func() error {
+		// provide a way to wait on wg.Wait() inside a select
+		done := make(chan bool)
+		go func() {
+			wg.Wait()
+			done <- true
+		}()
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-done:
+			close(pathC)
+			return nil
+		}
+	})
+
+	// provide eg.Wait() on a channel for returning
+	errC := make(chan error)
+	go func() { errC <- eg.Wait() }()
+
+	return resC, errC
+}
+
+// folderListWork takes input from pathC, lists the path, adds listed folders back into pathC, and
+// adds non-folders into results.
+func (c *Client) folderListWork(ctx context.Context, root string, wg *sync.WaitGroup, pathC, resC chan string) error {
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case path, ok := <-pathC:
+			if !ok {
+				return nil
+			}
+			if IsFolder(path) {
+				list, err := c.PathList(path)
+				if err != nil {
+					return newWrapErr(root, ErrFolderList, err)
+				}
+				for _, item := range list {
+					item = EnsurePrefix(item, path)
+					wg.Add(1)
+					go func(p string) { pathC <- p }(item)
+				}
+			} else {
+				if c.absolutepath {
+					resC <- path
+				} else {
+					resC <- strings.TrimPrefix(path, root)
+				}
+			}
+			wg.Done()
+		}
+	}
+}

vaku/folder_list_test.gobak → vaku/folder_list_test.go

@@ -1,6 +1,7 @@
 package vaku
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -24,15 +25,35 @@ func TestFolderList(t *testing.T) {
 			wantErr: nil,
 		},
 		{
-			name: "test/inner",
-			give: "test/inner",
+			name: "test/inner trailing slash",
+			give: "test/inner/",
 			want: []string{
 				"WKNC3muM",
 				"A2xlzTfE",
 				"again/inner/UCrt6sZT",
 			},
 			wantErr: nil,
 		},
+		{
+			name: "test/inner no slash and absolute path",
+			give: "test/inner",
+			want: []string{
+				"test/inner/WKNC3muM",
+				"test/inner/A2xlzTfE",
+				"test/inner/again/inner/UCrt6sZT",
+			},
+			giveOptions: []Option{WithAbsolutePath(true)},
+			wantErr:     nil,
+		},
+		{
+			name: "list error",
+			give: "test/inner",
+			want: []string{},
+			giveLogical: &errLogical{
+				err: errInject,
+			},
+			wantErr: []error{ErrFolderList, ErrPathList, ErrVaultList},
+		},
 	}
 
 	for _, tt := range tests {
@@ -47,10 +68,11 @@ func TestFolderList(t *testing.T) {
 			for _, ver := range kvMountVersions {
 				path := addMountToPath(t, tt.give, ver)
 
-				list, err := client.FolderList(path)
+				list, err := client.FolderList(context.Background(), path)
 				compareErrors(t, err, tt.wantErr)
 
-				assert.Equal(t, tt.want, list)
+				TrimListPrefix(list, ver)
+				assert.ElementsMatch(t, tt.want, list)
 			}
 		})
 	}

vaku/helpers.go

@@ -10,6 +10,19 @@ func IsFolder(p string) bool {
 	return strings.HasSuffix(p, "/")
 }
 
+// MakeFolder adds a slash to the end of a path if it doesn't already have one
+func MakeFolder(p string) string {
+	return KeyJoin(p, "/")
+}
+
+// EnsurePrefix adds a prefix to a path if it doesn't already have it
+func EnsurePrefix(p, pfx string) string {
+	if strings.HasPrefix(p, pfx) {
+		return p
+	}
+	return KeyJoin(pfx, p)
+}
+
 // KeyJoin combines strings into a clean Vault key. Keys may have a trailing '/' to signify they are a folder.
 func KeyJoin(k ...string) string {
 	if strings.HasSuffix(k[len(k)-1], "/") {

vaku/helpers_test.go

@@ -53,6 +53,100 @@ func TestIsFolder(t *testing.T) {
 	}
 }
 
+func TestMakeFolder(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		give string
+		want string
+	}{
+		{
+			give: "",
+			want: "/",
+		},
+		{
+			give: "a",
+			want: "a/",
+		},
+		{
+			give: "a/",
+			want: "a/",
+		},
+		{
+			give: "a/b",
+			want: "a/b/",
+		},
+		{
+			give: "a/",
+			want: "a/",
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.give, func(t *testing.T) {
+			t.Parallel()
+
+			assert.Equal(t, tt.want, MakeFolder(tt.give))
+		})
+	}
+}
+
+func TestEnsurePrefix(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		give       string
+		givePrefix string
+		want       string
+	}{
+		{
+			give:       "",
+			givePrefix: "",
+			want:       "",
+		},
+		{
+			give:       "a",
+			givePrefix: "",
+			want:       "a",
+		},
+		{
+			give:       "",
+			givePrefix: "a",
+			want:       "a",
+		},
+		{
+			give:       "a/",
+			givePrefix: "a",
+			want:       "a/",
+		},
+		{
+			give:       "a",
+			givePrefix: "a/",
+			want:       "a/a",
+		},
+		{
+			give:       "a/b/c/d",
+			givePrefix: "a/b/",
+			want:       "a/b/c/d",
+		},
+		{
+			give:       "a/b/c/d",
+			givePrefix: "b",
+			want:       "b/a/b/c/d",
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.give, func(t *testing.T) {
+			t.Parallel()
+
+			assert.Equal(t, tt.want, EnsurePrefix(tt.give, tt.givePrefix))
+		})
+	}
+}
+
 func TestKeyJoin(t *testing.T) {
 	t.Parallel()