Snapshots (#83)

Add ability to "snapshot" output for a single final state rather than incremental updates (eg, when piping the output)

README.md

@@ -26,7 +26,7 @@ See [rare.zdyn.net](https://rare.zdyn.net) or the [docs/ folder](docs/) for the
  * Aggregating and realtime summary (Don't have to wait for all data to be scanned)
  * Multi-threaded reading, parsing, and aggregation (It's fast)
  * Color-coded outputs (optionally)
- * Pipe support (stdin for reading, stdout will disable color) eg. `tail -f | rare ...`
+ * Pipe support (stdin for reading, stdout will disable realtime) eg. `tail -f | rare ... > out`
 
 Take a look at [examples](docs/usage/examples.md) to see more of what *rare* does.
 

cmd/analyze.go

@@ -16,7 +16,7 @@ func humanf(arg interface{}) string {
 	return color.Wrap(color.BrightWhite, humanize.Hf(arg))
 }
 
-func writeAggrOutput(writer *multiterm.TermWriter, aggr *aggregation.MatchNumerical, extra bool, quantiles []float64) int {
+func writeAggrOutput(writer multiterm.MultilineTerm, aggr *aggregation.MatchNumerical, extra bool, quantiles []float64) int {
 	writer.WriteForLinef(0, "Samples:  %v", color.Wrap(color.BrightWhite, humanize.Hi(aggr.Count())))
 	writer.WriteForLinef(1, "Mean:     %v", humanf(aggr.Mean()))
 	writer.WriteForLinef(2, "StdDev:   %v", humanf(aggr.StdDev()))
@@ -59,8 +59,7 @@ func analyzeFunction(c *cli.Context) error {
 	}
 
 	aggr := aggregation.NewNumericalAggregator(&config)
-	writer := multiterm.New()
-	defer multiterm.ResetCursor()
+	writer := helpers.BuildVTermFromArguments(c)
 
 	batcher := helpers.BuildBatcherFromArguments(c)
 	ext := helpers.BuildExtractorFromArguments(c, batcher)
@@ -102,6 +101,7 @@ func analyzeCommand() *cli.Command {
 				Usage:   "Adds a quantile to the output set. Requires --extra",
 				Value:   cli.NewStringSlice("90", "99", "99.9"),
 			},
+			helpers.SnapshotFlag,
 		},
 	})
 }

cmd/bargraph.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"rare/cmd/helpers"
 	"rare/pkg/aggregation"
-	"rare/pkg/multiterm"
 	"rare/pkg/multiterm/termrenderers"
 
 	"github.com/urfave/cli/v2"
@@ -20,8 +19,9 @@ func bargraphFunction(c *cli.Context) error {
 		sortName = c.String(helpers.DefaultSortFlag.Name)
 	)
 
+	vt := helpers.BuildVTermFromArguments(c)
 	counter := aggregation.NewSubKeyCounter()
-	writer := termrenderers.NewBarGraph(multiterm.New())
+	writer := termrenderers.NewBarGraph(vt)
 	writer.Stacked = stacked
 
 	batcher := helpers.BuildBatcherFromArguments(c)
@@ -63,6 +63,7 @@ func bargraphCommand() *cli.Command {
 				Usage:   "Display bargraph as stacked",
 			},
 			helpers.DefaultSortFlag,
+			helpers.SnapshotFlag,
 		},
 	})
 }

cmd/heatmap.go

@@ -32,7 +32,8 @@ func heatmapFunction(c *cli.Context) error {
 	rowSorter := helpers.BuildSorterOrFail(sortRows)
 	colSorter := helpers.BuildSorterOrFail(sortCols)
 
-	writer := termrenderers.NewHeatmap(multiterm.New(), numRows, numCols)
+	vt := helpers.BuildVTermFromArguments(c)
+	writer := termrenderers.NewHeatmap(vt, numRows, numCols)
 
 	writer.FixedMin = minFixed
 	writer.FixedMax = maxFixed
@@ -96,6 +97,7 @@ func heatmapCommand() *cli.Command {
 				Usage: helpers.DefaultSortFlag.Usage,
 				Value: helpers.DefaultSortFlag.Value,
 			},
+			helpers.SnapshotFlag,
 		},
 	})
 }

cmd/helpers/output.go

@@ -0,0 +1,25 @@
+package helpers
+
+import (
+	"rare/pkg/multiterm"
+	"rare/pkg/multiterm/termstate"
+
+	"github.com/urfave/cli/v2"
+)
+
+var SnapshotFlag = &cli.BoolFlag{
+	Name:  "snapshot",
+	Usage: "In aggregators that support it, only output final results, and not progressive updates. Will enable automatically when piping output",
+}
+
+func BuildVTerm(forceSnapshot bool) multiterm.MultilineTerm {
+	if forceSnapshot || termstate.IsPipedOutput() {
+		return multiterm.NewBufferedTerm()
+	}
+	return multiterm.New()
+}
+
+func BuildVTermFromArguments(c *cli.Context) multiterm.MultilineTerm {
+	snapshot := c.Bool(SnapshotFlag.Name)
+	return BuildVTerm(snapshot)
+}

cmd/helpers/output_test.go

@@ -0,0 +1,27 @@
+package helpers
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli/v2"
+)
+
+func TestBuildVTerm(t *testing.T) {
+	assert.NotNil(t, BuildVTerm(false))
+	assert.NotNil(t, BuildVTerm(true))
+}
+
+func TestBuildVTermFromArgs(t *testing.T) {
+	app := cli.NewApp()
+	app.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name: "snapshot",
+		},
+	}
+	app.Action = func(ctx *cli.Context) error {
+		BuildVTermFromArguments(ctx)
+		return nil
+	}
+	app.Run([]string{"", "--snapshot"})
+}

cmd/helpers/updatingAggregator.go

@@ -1,13 +1,11 @@
 package helpers
 
 import (
-	"fmt"
 	"os"
 	"os/signal"
 	"rare/pkg/aggregation"
 	"rare/pkg/extractor"
 	"rare/pkg/logger"
-	"rare/pkg/multiterm"
 	"sync"
 	"time"
 )
@@ -18,7 +16,6 @@ import (
 //   writeOutput - triggered after a delay, only if there's an update
 // The two functions are guaranteed to never happen at the same time
 func RunAggregationLoop(ext *extractor.Extractor, aggregator aggregation.Aggregator, writeOutput func()) {
-	defer multiterm.ResetCursor()
 	logger.DeferLogs()
 
 	// Updater sync variables
@@ -62,5 +59,4 @@ PROCESSING_LOOP:
 	outputDone <- true
 
 	writeOutput()
-	fmt.Println()
 }

cmd/histo.go

@@ -35,8 +35,9 @@ func histoFunction(c *cli.Context) error {
 		sortName = c.String(helpers.DefaultSortFlag.Name)
 	)
 
+	vt := helpers.BuildVTermFromArguments(c)
 	counter := aggregation.NewCounter()
-	writer := termrenderers.NewHistogram(multiterm.New(), topItems)
+	writer := termrenderers.NewHistogram(vt, topItems)
 	writer.ShowBar = c.Bool("bars") || extra
 	writer.ShowPercentage = c.Bool("percentage") || extra
 
@@ -56,6 +57,8 @@ func histoFunction(c *cli.Context) error {
 		writer.WriteFooter(1, batcher.StatusString())
 	})
 
+	// Not deferred because of the `all` below to print out before it
+	// when in snapshot mode
 	writer.Close()
 
 	if all {
@@ -118,6 +121,7 @@ func histogramCommand() *cli.Command {
 				Value: 0,
 			},
 			helpers.DefaultSortFlagWithDefault("value"),
+			helpers.SnapshotFlag,
 		},
 	})
 }

cmd/histo_test.go

@@ -15,8 +15,8 @@ func TestHistogram(t *testing.T) {
 }
 
 func TestHistogramRender(t *testing.T) {
-	out, eout, err := testCommandCapture(histogramCommand(), `-m "(\d+)" -e "{bucket {1} 10}" testdata/log.txt`)
+	out, eout, err := testCommandCapture(histogramCommand(), `--snapshot -m "(\d+)" -e "{bucket {1} 10}" testdata/log.txt`)
 	assert.NoError(t, err)
-	assert.Contains(t, out, "Matched: 3 / 6 (Groups: 2)")
+	assert.Equal(t, out, "0                   2         \n20                  1         \n\n\n\nMatched: 3 / 6 (Groups: 2)\n96 B (0 B/s) \n")
 	assert.Equal(t, "", eout)
 }

cmd/tabulate.go

@@ -7,7 +7,6 @@ import (
 	"rare/pkg/color"
 	"rare/pkg/expressions"
 	"rare/pkg/humanize"
-	"rare/pkg/multiterm"
 	"rare/pkg/multiterm/termrenderers"
 
 	"github.com/urfave/cli/v2"
@@ -32,7 +31,8 @@ func tabulateFunction(c *cli.Context) error {
 	)
 
 	counter := aggregation.NewTable(delim)
-	writer := termrenderers.NewTable(multiterm.New(), numCols+2, numRows+2)
+	vt := helpers.BuildVTermFromArguments(c)
+	writer := termrenderers.NewTable(vt, numCols+2, numRows+2)
 
 	batcher := helpers.BuildBatcherFromArguments(c)
 	ext := helpers.BuildExtractorFromArguments(c, batcher)
@@ -148,6 +148,7 @@ func tabulateCommand() *cli.Command {
 				Usage: helpers.DefaultSortFlag.Usage,
 				Value: "value",
 			},
+			helpers.SnapshotFlag,
 		},
 	})
 }

docs/index.md

@@ -24,7 +24,7 @@ Supports various CLI-based graphing and metric formats (filter (grep-like), hist
  * Aggregating and realtime summary (Don't have to wait for all data to be scanned)
  * Multi-threaded reading, parsing, and aggregation (It's fast)
  * Color-coded outputs (optionally)
- * Pipe support (stdin for reading, stdout will disable color) eg. `tail -f | rare ...`
+ * Pipe support (stdin for reading, stdout will disable realtime) eg. `tail -f | rare ... > out`
 
 Take a look at [examples](usage/examples.md) to see more of what *rare* does.
 

pkg/color/coloring.go

@@ -3,7 +3,7 @@ package color
 import (
 	"fmt"
 	"io"
-	"os"
+	"rare/pkg/multiterm/termstate"
 	"strings"
 	"unicode/utf8"
 )
@@ -56,10 +56,8 @@ var Enabled = true
 var GroupColors = [...]ColorCode{Red, Green, Yellow, Blue, Magenta, Cyan, BrightRed, BrightGreen, BrightYellow, BrightBlue, BrightMagenta, BrightCyan}
 
 func init() {
-	if fi, err := os.Stdout.Stat(); err == nil {
-		if (fi.Mode() & os.ModeCharDevice) == 0 {
-			Enabled = false
-		}
+	if termstate.IsPipedOutput() {
+		Enabled = false
 	}
 }
 

pkg/multiterm/bufferedterm.go

@@ -0,0 +1,19 @@
+package multiterm
+
+import "os"
+
+type BufferedTerm struct {
+	*VirtualTerm
+}
+
+// NewBufferedTerm writes on Close
+func NewBufferedTerm() *BufferedTerm {
+	return &BufferedTerm{
+		NewVirtualTerm(),
+	}
+}
+
+func (s *BufferedTerm) Close() {
+	s.WriteToOutput(os.Stdout)
+	s.VirtualTerm.Close()
+}

pkg/multiterm/bufferedterm_test.go

@@ -0,0 +1,11 @@
+package multiterm
+
+import (
+	"testing"
+)
+
+func TestBufferedTerm(t *testing.T) {
+	vt := NewBufferedTerm()
+	vt.WriteForLine(0, "hello")
+	vt.Close()
+}

pkg/multiterm/cursor.go

@@ -36,7 +36,3 @@ func showCursor() {
 func eraseRemainingLine() {
 	fmt.Print(escape("[0K"))
 }
-
-func ResetCursor() {
-	showCursor()
-}

pkg/multiterm/intf.go

@@ -2,5 +2,6 @@ package multiterm
 
 type MultilineTerm interface {
 	WriteForLine(line int, s string)
+	WriteForLinef(line int, format string, args ...interface{})
 	Close()
 }

pkg/multiterm/linetrim.go

@@ -2,9 +2,7 @@ package multiterm
 
 import (
 	"io"
-	"os"
-
-	"golang.org/x/term"
+	"rare/pkg/multiterm/termstate"
 )
 
 var AutoTrim = true
@@ -13,22 +11,8 @@ const defaultRows, defaultCols = 24, 80
 
 var computedRows, computedCols = 0, 0
 
-func getTermRowsCols() (rows, cols int, ok bool) {
-	fd := int(os.Stdout.Fd())
-	if !term.IsTerminal(fd) {
-		return 0, 0, false
-	}
-
-	cols, rows, err := term.GetSize(fd)
-	if err != nil {
-		return 0, 0, false
-	}
-
-	return rows, cols, true
-}
-
 func init() {
-	if rows, cols, ok := getTermRowsCols(); ok {
+	if rows, cols, ok := termstate.GetTermRowsCols(); ok {
 		computedRows, computedCols = rows, cols
 	} else {
 		AutoTrim = false

pkg/multiterm/multiterm.go

@@ -41,6 +41,7 @@ func (s *TermWriter) WriteForLine(line int, text string) {
 
 func (s *TermWriter) Close() {
 	s.goTo(s.maxLine)
+	fmt.Println() // Put cursor after last line
 	if s.cursorHidden {
 		showCursor()
 	}

pkg/multiterm/multiterm_test.go

@@ -9,4 +9,6 @@ func TestBasicMultiterm(t *testing.T) {
 	mt.WriteForLine(0, "Hello")
 	mt.WriteForLine(1, "you")
 	mt.WriteForLine(10, "There")
+	mt.WriteForLinef(5, "This is %s", "Test")
+	mt.Close()
 }

pkg/multiterm/termstate/term.go

@@ -0,0 +1,32 @@
+package termstate
+
+import (
+	"os"
+
+	"golang.org/x/term"
+)
+
+// Returns 'true' if output is being piped (Not char device)
+func IsPipedOutput() bool {
+	if fi, err := os.Stdout.Stat(); err == nil {
+		if (fi.Mode() & os.ModeCharDevice) == 0 {
+			return true
+		}
+	}
+	return false
+}
+
+// Gets size of the terminal
+func GetTermRowsCols() (rows, cols int, ok bool) {
+	fd := int(os.Stdout.Fd())
+	if !term.IsTerminal(fd) {
+		return 0, 0, false
+	}
+
+	cols, rows, err := term.GetSize(fd)
+	if err != nil {
+		return 0, 0, false
+	}
+
+	return rows, cols, true
+}