Skip to content

Commit

Permalink
feat(bundle): optionally displays labels associated to a bundle
Browse files Browse the repository at this point in the history
* fixes oneconcern#235

Signed-off-by: Frederic BIDON <frederic@oneconcern.com>
  • Loading branch information
fredbi committed Dec 1, 2022
1 parent 31116aa commit 4a1072c
Show file tree
Hide file tree
Showing 9 changed files with 182 additions and 9 deletions.
35 changes: 35 additions & 0 deletions cmd/datamon/cmd/bundle.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"bytes"
"context"
"fmt"
"strings"
"text/template"

context2 "github.com/oneconcern/datamon/pkg/context"
Expand Down Expand Up @@ -33,16 +34,21 @@ together.`,
},
}

<<<<<<< HEAD
var (
useBundleTemplate func(flagsT) *template.Template
bundleDescriptorTemplate func(flagsT) *template.Template
)

=======
>>>>>>> feat(bundle): optionally displays labels associated to a bundle
func init() {
addTemplateFlag(bundleCmd)
addSkipAuthFlag(bundleCmd, true)
rootCmd.AddCommand(bundleCmd)
}

<<<<<<< HEAD
bundleDescriptorTemplate = func(opts flagsT) *template.Template {
if opts.core.Template != "" {
t, err := template.New("list line").Parse(datamonFlags.core.Template)
Expand All @@ -59,6 +65,20 @@ func init() {
const useBundleTemplateString = `Using bundle: {{.ID}}`
return template.Must(template.New("use bundle").Parse(useBundleTemplateString))
}
=======
func bundleDescriptorTemplate(withLabels bool) *template.Template {
// bundle rendering comes in 2 flavors: one without the labels field, the other with the
// comma separated list of labels set on the bundle. When the label field is wanted,
// but empty, it takes the <no label> value.
var listLineTemplateString string
if !withLabels {
listLineTemplateString = `{{.ID}} , {{.Timestamp}} , {{.Message}}`
} else {
// template with labels
listLineTemplateString = `{{.ID}} ,{{.Labels}}, {{.Timestamp}} , {{.Message}}`
}
return template.Must(template.New("list line").Parse(listLineTemplateString))
>>>>>>> feat(bundle): optionally displays labels associated to a bundle
}

func setLatestOrLabelledBundle(ctx context.Context, remote context2.Stores) error {
Expand Down Expand Up @@ -111,3 +131,18 @@ func getConcurrencyFactor(batchSize int) int {
}
return concurrency
}

// displayBundleLabels constructs a string representation of a list of labels associated to a bundle
func displayBundleLabels(bundleID string, labels []model.LabelDescriptor) string {
bundleLabels := make([]string, 0, len(labels))
for _, label := range labels {
if label.BundleID == bundleID {
bundleLabels = append(bundleLabels, label.Name)
}
}
if len(bundleLabels) > 0 {
// using ";" to keep simple split rule on main fields
return "(" + strings.Join(bundleLabels, ";") + ")"
}
return "<no label>"
}
23 changes: 23 additions & 0 deletions cmd/datamon/cmd/bundle_get.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/oneconcern/datamon/pkg/core"
status "github.com/oneconcern/datamon/pkg/core/status"
"github.com/oneconcern/datamon/pkg/errors"
"github.com/oneconcern/datamon/pkg/model"

"github.com/spf13/cobra"
"golang.org/x/sys/unix"
Expand Down Expand Up @@ -69,8 +70,29 @@ exits with ENOENT status otherwise.`,
return
}

var labels []model.LabelDescriptor
if datamonFlags.bundle.WithLabels {
// optionally starts by retrieving labels on this repo
labels = getLabels(remoteStores)
}

var buf bytes.Buffer
<<<<<<< HEAD
err = bundleDescriptorTemplate(datamonFlags).Execute(&buf, bundle.BundleDescriptor)
=======
if labels != nil {
data := struct {
model.BundleDescriptor
Labels string
}{
BundleDescriptor: bundle.BundleDescriptor,
}
data.Labels = displayBundleLabels(bundle.BundleDescriptor.ID, labels)
err = bundleDescriptorTemplate(true).Execute(&buf, data)
} else {
err = bundleDescriptorTemplate(false).Execute(&buf, bundle.BundleDescriptor)
}
>>>>>>> feat(bundle): optionally displays labels associated to a bundle
if err != nil {
wrapFatalln("executing template", err)
}
Expand All @@ -90,6 +112,7 @@ func init() {

addBundleFlag(GetBundleCommand)
addLabelNameFlag(GetBundleCommand)
addWithLabelFlag(GetBundleCommand)

bundleCmd.AddCommand(GetBundleCommand)
}
40 changes: 37 additions & 3 deletions cmd/datamon/cmd/bundle_list.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,40 @@ import (
"github.com/spf13/cobra"
)

<<<<<<< HEAD
func applyBundleTemplate(bundle model.BundleDescriptor) error {
var buf bytes.Buffer
if err := bundleDescriptorTemplate(datamonFlags).Execute(&buf, bundle); err != nil {
// NOTE(frederic): to be discussed - PR#267 introduced a change here
// by stopping upon errors while it was previously non-blocking
return fmt.Errorf("executing template: %w", err)
=======
func applyBundleTemplate(labels []model.LabelDescriptor) func(model.BundleDescriptor) error {
return func(bundle model.BundleDescriptor) error {
var (
buf bytes.Buffer
err error
)

if labels != nil {
data := struct {
model.BundleDescriptor
Labels string
}{
BundleDescriptor: bundle,
}
data.Labels = displayBundleLabels(bundle.ID, labels)
err = bundleDescriptorTemplate(true).Execute(&buf, data)
} else {
err = bundleDescriptorTemplate(false).Execute(&buf, bundle)
}
if err != nil {
return fmt.Errorf("executing template: %w", err)
}
log.Println(buf.String())
return nil
>>>>>>> feat(bundle): optionally displays labels associated to a bundle
}
log.Println(buf.String())
return nil
}

// BundleListCommand describes the CLI command for listing bundles
Expand All @@ -47,7 +72,15 @@ This is analogous to the "git log" command. The bundle ID works like a git commi
wrapFatalln("create remote stores", err)
return
}
err = core.ListBundlesApply(datamonFlags.repo.RepoName, remoteStores, applyBundleTemplate,

var labels []model.LabelDescriptor
if datamonFlags.bundle.WithLabels {
// optionally starts by retrieving labels on this repo
labels = getLabels(remoteStores)
}

err = core.ListBundlesApply(datamonFlags.repo.RepoName, remoteStores,
applyBundleTemplate(labels),
core.ConcurrentList(datamonFlags.core.ConcurrencyFactor),
core.BatchSize(datamonFlags.core.BatchSize),
core.WithMetrics(datamonFlags.root.metrics.IsEnabled()),
Expand All @@ -71,6 +104,7 @@ func init() {

addCoreConcurrencyFactorFlag(BundleListCommand, 500)
addBatchSizeFlag(BundleListCommand)
addWithLabelFlag(BundleListCommand)

bundleCmd.AddCommand(BundleListCommand)
}
52 changes: 46 additions & 6 deletions cmd/datamon/cmd/cli_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -418,6 +418,7 @@ type bundleListEntry struct {
hash string
message string
time time.Time
labels string
}

type bundleListEntries []bundleListEntry
Expand All @@ -435,16 +436,17 @@ func (b bundleListEntries) Last() bundleListEntry {
return b[len(b)-1]
}

func listBundles(t *testing.T, repoName string) (bundleListEntries, error) {
func listBundles(t *testing.T, repoName string, flags ...string) (bundleListEntries, error) {
r, w, err := os.Pipe()
if err != nil {
panic(err)
}
log.SetOutput(w)
runCmd(t, []string{"bundle",
runCmd(t, append([]string{"bundle",
"list",
"--repo", repoName,
}, "list bundles", false)
}, flags...),
"list bundles", false)
log.SetOutput(os.Stdout)
w.Close()
//
Expand All @@ -453,16 +455,31 @@ func listBundles(t *testing.T, repoName string) (bundleListEntries, error) {
bles := make(bundleListEntries, 0)
for _, line := range getDataLogLines(t, string(lb), []string{}) {
sl := strings.Split(line, ",")
t, err := time.Parse(timeForm, strings.TrimSpace(sl[1]))

// updates expectations about format, depending on flags
expectLabels := false
offset := 0
for _, flag := range flags {
if flag == "--with-labels" {
expectLabels = true
offset = 1
break
}
}

t, err := time.Parse(timeForm, strings.TrimSpace(sl[1+offset]))
if err != nil {
return nil, err
}
rle := bundleListEntry{
rawLine: line,
hash: strings.TrimSpace(sl[0]), // bundle ID
message: strings.TrimSpace(sl[2]),
message: strings.TrimSpace(sl[2+offset]),
time: t,
}
if expectLabels {
rle.labels = strings.TrimSpace(sl[1])
}
bles = append(bles, rle)
}
// bundles are ordered by lexicographic order of bundle IDs
Expand Down Expand Up @@ -570,6 +587,7 @@ func TestGetLabel(t *testing.T) {
"--description", "testing",
"--repo", repo1,
}, "create first test repo", false)

runCmd(t, []string{"label",
"get",
"--repo", repo1,
Expand All @@ -579,6 +597,7 @@ func TestGetLabel(t *testing.T) {
"ENOENT on nonexistant label")
files := testUploadTrees[0]
file := files[0]

runCmd(t, []string{"bundle",
"upload",
"--path", dirPathStr(t, file),
Expand All @@ -587,11 +606,26 @@ func TestGetLabel(t *testing.T) {
"--label", label,
"--concurrency-factor", concurrencyFactor,
}, "upload bundle at "+dirPathStr(t, file), false)

runCmd(t, []string{"label",
"get",
"--repo", repo1,
"--label", label,
}, "get label", false)

// query bundles with labels
bundles, err := listBundles(t, repo1, "--with-labels")
require.NoError(t, err)
bundle := bundles.Last()
require.Equal(t, "("+label+")", bundle.labels)

runCmd(t, []string{"bundle",
"get",
"--repo", repo1,
"--bundle", bundle.hash,
"--with-labels",
}, "("+label+")", false)
require.NoError(t, err)
}

type labelListEntry struct {
Expand Down Expand Up @@ -709,7 +743,13 @@ func TestSetLabel(t *testing.T) {
ll = listLabels(t, repo1, "")
require.Equal(t, len(ll), 0, "no labels created yet")

bundles, err := listBundles(t, repo1)
// query bundles with labels when no label set
bundles, err := listBundles(t, repo1, "--with-labels")
require.NoError(t, err)
bundle := bundles.Last()
require.Equal(t, "<no label>", bundle.labels)

bundles, err = listBundles(t, repo1)
require.NoError(t, err, "error out of listBundles() test helper")
require.Equal(t, 1, bundles.Len(), "bundle count in test repo")

Expand Down
13 changes: 13 additions & 0 deletions cmd/datamon/cmd/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ type flagsT struct {
SkipOnError bool
ConcurrencyFactor int
NameFilter string
<<<<<<< HEAD
ForceDest bool
}
fs struct {
Expand All @@ -47,6 +48,12 @@ type flagsT struct {
WithVerifyHash bool
WithVerifyBlobHash bool
WithRetry bool
=======
CacheSize flagext.ByteSize
WithPrefetch int
WithVerifyHash bool
WithLabels bool
>>>>>>> feat(bundle): optionally displays labels associated to a bundle
}
web struct {
port int
Expand Down Expand Up @@ -280,6 +287,12 @@ func addLabelPrefixFlag(cmd *cobra.Command) string {
return prefixString
}

func addWithLabelFlag(cmd *cobra.Command) string {
withLabels := "with-labels"
cmd.Flags().BoolVar(&datamonFlags.bundle.WithLabels, withLabels, false, "Include labels in the returned bundle metadata")
return withLabels
}

func addRepoNameOptionFlag(cmd *cobra.Command) string {
const repo = "repo"
if cmd != nil {
Expand Down
14 changes: 14 additions & 0 deletions cmd/datamon/cmd/label.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ package cmd
import (
"text/template"

context2 "github.com/oneconcern/datamon/pkg/context"
"github.com/oneconcern/datamon/pkg/core"
"github.com/oneconcern/datamon/pkg/model"
"github.com/spf13/cobra"
)

Expand Down Expand Up @@ -47,3 +50,14 @@ func init() {
return template.Must(template.New("list line").Parse(listLineTemplateString))
}
}

// getLabels is a helper to synchronously retrieve labels and enrich the result from other commands
func getLabels(remoteStores context2.Stores) []model.LabelDescriptor {
labels, err := core.ListLabels(datamonFlags.repo.RepoName, remoteStores, "",
core.ConcurrentList(datamonFlags.core.ConcurrencyFactor),
core.BatchSize(datamonFlags.core.BatchSize))
if err != nil {
wrapFatalln("could not download label list: %w", err)
}
return labels
}
8 changes: 8 additions & 0 deletions docs/usage/datamon_bundle_get.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,18 @@ datamon bundle get [flags]
### Options

```
<<<<<<< HEAD
--bundle string The hash id for the bundle, if not specified the latest bundle will be used
-h, --help help for get
--label string The human-readable name of a label
--repo (*) string The name of this repository
=======
--bundle string The hash id for the bundle, if not specified the latest bundle will be used
-h, --help help for get
--label string The human-readable name of a label
--repo string The name of this repository
--with-labels Include labels in the returned bundle metadata
>>>>>>> feat(bundle): optionally displays labels associated to a bundle
```

### Options inherited from parent commands
Expand Down
5 changes: 5 additions & 0 deletions docs/usage/datamon_bundle_list.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,12 @@ datamon bundle list [flags]
--batch-size int Number of bundles streamed together as a batch. This can be tuned for performance based on network connectivity (default 1024)
--concurrency-factor int Heuristic on the amount of concurrency used by core operations. Concurrent retrieval of metadata is capped by the 'batch-size' parameter. Turn this value down to use less memory, increase for faster operations. (default 500)
-h, --help help for list
<<<<<<< HEAD
--repo (*) string The name of this repository
=======
--repo string The name of this repository
--with-labels Include labels in the returned bundle metadata
>>>>>>> feat(bundle): optionally displays labels associated to a bundle
```

### Options inherited from parent commands
Expand Down
Loading

0 comments on commit 4a1072c

Please sign in to comment.