Redo tempo-cli with basic command structure and improvements (#385)

* Convert tempo-cli to kong cli parser, add some basic command structure

* Rename commands to 'list block' 'list bucket'

* Update list bucket command to put index analysis behind --load-index flag, print results more dynamically

* Rename list bucket to list blocks, improve parameter descriptions and locations

* vendor-check fixes

* ignore tempo-cli binary

* Fix lint re: unused values

* Add tempo-cli section to docs website.  Update readme.md to point to docs for tempo-cli

* Apply suggestions from code review

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

* Apply suggestions from code review

* Update changelog

* Wording tweak based on feedback

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

.gitignore

@@ -5,4 +5,5 @@
 .idea
 .vscode
 /dist
+/cmd/tempo-cli/tempo-cli
 /example/docker-compose/example-data/tempo

CHANGELOG.md

@@ -1,5 +1,7 @@
 ## master / unreleased
 
+* [CHANGE] Redo tempo-cli with basic command structure and improvements [#385](https://github.com/grafana/tempo/pull/385)
+
 ## v0.4.0
 
 * [CHANGE] From path.Join to filepath.Join [#338](https://github.com/grafana/tempo/pull/338)

README.md

@@ -42,18 +42,7 @@ tempo-query is jaeger-query with a [hashicorp go-plugin](https://github.com/jaeg
 tempo-vulture is tempo's bird themed consistency checking tool.  It queries Loki, extracts trace ids and then queries tempo.  It metrics 404s and traces with missing spans.
 
 ### tempo-cli
-tempo-cli is the place to put any utility functionality related to tempo.
-
-Currently, it supports dumping header information for all blocks from gcs/s3.
-```
-go run ./cmd/tempo-cli -backend=gcs -bucket ops-tools-tracing-ops -tenant-id single-tenant
-```
-
-It also supports connecting to tempo directly to get a trace result in JSON.
-```console
-$ go run cmd/tempo-cli/main.go -query-endpoint http://localhost:3100 -traceID 2a61c34ff39a1518 -orgID 1
-{"batches":[{"resource":{"attributes":[{"key":"service.name","value":{"Value":{"string_value":"cortex-ingester"}}}.....}
-```
+tempo-cli is the place to put any utility functionality related to tempo.  See [Documentation](https://grafana.com/docs/tempo/latest/cli/) for more info.
 
 
 ## TempoDB

cmd/tempo-cli/cmd-list-block.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/google/uuid"
+	tempodb_backend "github.com/grafana/tempo/tempodb/backend"
+	"github.com/grafana/tempo/tempodb/encoding"
+)
+
+type listBlockCmd struct {
+	backendOptions
+
+	TenantID   string `arg:"" help:"tenant-id within the bucket"`
+	BlockID    string `arg:"" help:"block ID to list"`
+	CheckDupes bool   `help:"check contents of block for duplicate trace IDs (warning, can be intense)"`
+}
+
+func (cmd *listBlockCmd) Run(ctx *globalOptions) error {
+	r, c, err := loadBackend(&cmd.backendOptions, ctx)
+	if err != nil {
+		return err
+	}
+
+	return dumpBlock(r, c, cmd.TenantID, time.Hour, cmd.BlockID, cmd.CheckDupes)
+}
+
+func dumpBlock(r tempodb_backend.Reader, c tempodb_backend.Compactor, tenantID string, windowRange time.Duration, blockID string, checkDupes bool) error {
+	id := uuid.MustParse(blockID)
+
+	meta, err := r.BlockMeta(context.TODO(), id, tenantID)
+	if err != nil && err != tempodb_backend.ErrMetaDoesNotExist {
+		return err
+	}
+
+	compactedMeta, err := c.CompactedBlockMeta(id, tenantID)
+	if err != nil && err != tempodb_backend.ErrMetaDoesNotExist {
+		return err
+	}
+
+	if meta == nil && compactedMeta == nil {
+		fmt.Println("Unable to load any meta for block", blockID)
+		return nil
+	}
+
+	objects, lvl, window, start, end := blockStats(meta, compactedMeta, windowRange)
+
+	fmt.Println("ID            : ", id)
+	fmt.Println("Total Objects : ", objects)
+	fmt.Println("Level         : ", lvl)
+	fmt.Println("Window        : ", window)
+	fmt.Println("Start         : ", start)
+	fmt.Println("End           : ", end)
+
+	if checkDupes {
+		fmt.Println("Searching for dupes ...")
+
+		iter, err := encoding.NewBackendIterator(tenantID, id, 10*1024*1024, r)
+		if err != nil {
+			return err
+		}
+
+		i := 0
+		dupe := 0
+		prevID := make([]byte, 16)
+		for {
+			objID, _, err := iter.Next()
+			if err == io.EOF {
+				break
+			} else if err != nil {
+				return err
+			}
+
+			if bytes.Equal(objID, prevID) {
+				dupe++
+			}
+
+			copy(prevID, objID)
+			i++
+			if i%100000 == 0 {
+				fmt.Println("Record: ", i)
+			}
+		}
+
+		fmt.Println("total: ", i)
+		fmt.Println("dupes: ", dupe)
+	}
+
+	return nil
+}

cmd/tempo-cli/cmd-list-blocks.go

@@ -0,0 +1,189 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"sort"
+	"strconv"
+	"time"
+
+	"github.com/google/uuid"
+	tempodb_backend "github.com/grafana/tempo/tempodb/backend"
+	"github.com/grafana/tempo/tempodb/encoding"
+	"github.com/olekukonko/tablewriter"
+)
+
+type listBlocksCmd struct {
+	TenantID  string `arg:"" help:"tenant-id within the bucket"`
+	LoadIndex bool   `help:"load block indexes and display additional information"`
+
+	backendOptions
+}
+
+func (l *listBlocksCmd) Run(ctx *globalOptions) error {
+	r, c, err := loadBackend(&l.backendOptions, ctx)
+	if err != nil {
+		return err
+	}
+
+	windowDuration := time.Hour
+
+	results, err := loadBucket(r, c, l.TenantID, windowDuration, l.LoadIndex)
+	if err != nil {
+		return err
+	}
+
+	displayResults(results, windowDuration, l.LoadIndex)
+	return nil
+}
+
+type bucketStats struct {
+	id              uuid.UUID
+	compactionLevel uint8
+	objects         int
+	window          int64
+	start           time.Time
+	end             time.Time
+
+	totalIDs     int
+	duplicateIDs int
+}
+
+func loadBucket(r tempodb_backend.Reader, c tempodb_backend.Compactor, tenantID string, windowRange time.Duration, loadIndex bool) ([]bucketStats, error) {
+	blockIDs, err := r.Blocks(context.Background(), tenantID)
+	if err != nil {
+		return nil, err
+	}
+
+	fmt.Println("total blocks: ", len(blockIDs))
+	results := make([]bucketStats, 0)
+
+	for _, id := range blockIDs {
+		fmt.Print(".")
+
+		meta, err := r.BlockMeta(context.Background(), id, tenantID)
+		if err != nil && err != tempodb_backend.ErrMetaDoesNotExist {
+			return nil, err
+		}
+
+		compactedMeta, err := c.CompactedBlockMeta(id, tenantID)
+		if err != nil && err != tempodb_backend.ErrMetaDoesNotExist {
+			return nil, err
+		}
+
+		totalIDs := -1
+		duplicateIDs := -1
+
+		if loadIndex {
+			indexBytes, err := r.Index(context.Background(), id, tenantID)
+			if err == nil {
+				records, err := encoding.UnmarshalRecords(indexBytes)
+				if err != nil {
+					return nil, err
+				}
+				duplicateIDs = 0
+				totalIDs = len(records)
+				for i := 1; i < len(records); i++ {
+					if bytes.Equal(records[i-1].ID, records[i].ID) {
+						duplicateIDs++
+					}
+				}
+			}
+		}
+
+		objects, lvl, window, start, end := blockStats(meta, compactedMeta, windowRange)
+
+		results = append(results, bucketStats{
+			id:              id,
+			compactionLevel: lvl,
+			objects:         objects,
+			window:          window,
+			start:           start,
+			end:             end,
+
+			totalIDs:     totalIDs,
+			duplicateIDs: duplicateIDs,
+		})
+	}
+
+	sort.Slice(results, func(i, j int) bool {
+		bI := results[i]
+		bJ := results[j]
+
+		if bI.window == bJ.window {
+			return bI.compactionLevel < bJ.compactionLevel
+		}
+
+		return bI.window < bJ.window
+	})
+
+	return results, nil
+}
+
+func displayResults(results []bucketStats, windowDuration time.Duration, includeIndexInfo bool) {
+
+	columns := []string{"id", "lvl", "count", "window", "start", "end", "duration"}
+	if includeIndexInfo {
+		columns = append(columns, "idx", "dupe")
+	}
+
+	totalObjects := 0
+	out := make([][]string, 0)
+	for _, r := range results {
+
+		line := make([]string, 0)
+
+		for _, c := range columns {
+			s := ""
+			switch c {
+			case "id":
+				s = r.id.String()
+			case "lvl":
+				s = strconv.Itoa(int(r.compactionLevel))
+			case "count":
+				s = strconv.Itoa(r.objects)
+			case "window":
+				// Display compaction window in human-readable format
+				window := time.Unix(r.window*int64(windowDuration.Seconds()), 0).UTC()
+				s = window.Format(time.RFC3339)
+			case "start":
+				s = r.start.Format(time.RFC3339)
+			case "end":
+				s = r.end.Format(time.RFC3339)
+			case "duration":
+				// Time range included in bucket
+				s = fmt.Sprint(r.end.Sub(r.start).Round(time.Second))
+			case "idx":
+				// Number of entries in the index (may not be the same as the block when index downsampling enabled)
+				s = strconv.Itoa(r.totalIDs)
+			case "dupe":
+				// Number of duplicate IDs found in the index
+				s = strconv.Itoa(r.duplicateIDs)
+			}
+
+			line = append(line, s)
+		}
+
+		out = append(out, line)
+		totalObjects += r.objects
+	}
+
+	footer := make([]string, 0)
+	for _, c := range columns {
+		switch c {
+		case "count":
+			footer = append(footer, strconv.Itoa(totalObjects))
+		default:
+			footer = append(footer, "")
+		}
+	}
+
+	fmt.Println()
+	w := tablewriter.NewWriter(os.Stdout)
+	w.SetHeader(columns)
+	w.SetFooter(footer)
+	w.AppendBulk(out)
+	w.Render()
+}

cmd/tempo-cli/cmd-query.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/grafana/tempo/pkg/util"
+)
+
+type queryCmd struct {
+	APIEndpoint string `arg:"" help:"tempo api endpoint"`
+	TraceID     string `arg:"" help:"trace ID to retrieve"`
+
+	OrgID string `help:"optional orgID"`
+}
+
+func (cmd *queryCmd) Run(_ *globalOptions) error {
+
+	// util.QueryTrace will only add orgID header if len(orgID) > 0
+	trace, err := util.QueryTrace(cmd.APIEndpoint, cmd.TraceID, cmd.OrgID)
+	if err != nil {
+		return err
+	}
+
+	traceJSON, err := json.Marshal(trace)
+	if err != nil {
+		return err
+	}
+
+	fmt.Println(string(traceJSON))
+	return nil
+}