Heatmap (#73)

Add heatmap render method, optimize virtual-terminal (testing and some output), and more docs/testing

README.md

@@ -18,7 +18,7 @@ See [rare.zdyn.net](https://rare.zdyn.net) or the [docs/ folder](docs/) for the
 
 ## Features
 
- * Multiple summary formats including: filter (like grep), histogram, bar graphs, and numerical analysis
+ * Multiple summary formats including: filter (like grep), histogram, bar graphs, tables, heatmaps, and numerical analysis
  * File glob expansions (eg `/var/log/*` or `/var/log/*/*.log`) and `-R`
  * Optional gzip decompression (with `-z`)
  * Following `-f` or re-open following `-F` (use `--poll` to poll, and `--tail` to tail)
@@ -37,6 +37,7 @@ Output formats include:
 * `filter` is grep-like, in that each line will be processed and the extracted key will be output directly to stdout
 * `histogram` will count instances of the extracted key
 * `table` will count the key in 2 dimensions
+* `heatmap` will display a color-coded version of the strength of a cell in a dense format
 * `bargraph` will create either a stacked or non-stacked bargraph based on 2 dimensions
 * `analyze` will use the key as a numeric value and compute mean/median/mode/stddev/percentiles
 

cmd/commands.go

@@ -5,6 +5,7 @@ import "github.com/urfave/cli"
 var commands []cli.Command = []cli.Command{
 	*filterCommand(),
 	*histogramCommand(),
+	*heatmapCommand(),
 	*bargraphCommand(),
 	*analyzeCommand(),
 	*tabulateCommand(),

cmd/heatmap.go

@@ -0,0 +1,85 @@
+package cmd
+
+import (
+	"fmt"
+	"rare/cmd/helpers"
+	"rare/pkg/aggregation"
+	"rare/pkg/color"
+	"rare/pkg/expressions"
+	"rare/pkg/multiterm"
+	"rare/pkg/multiterm/termrenderers"
+
+	"github.com/urfave/cli"
+)
+
+func heatmapFunction(c *cli.Context) error {
+	var (
+		delim    = c.String("delim")
+		numRows  = c.Int("num")
+		numCols  = c.Int("cols")
+		minFixed = c.IsSet("min")
+		minVal   = c.Int64("min")
+		maxFixed = c.IsSet("max")
+		maxVal   = c.Int64("max")
+	)
+
+	counter := aggregation.NewTable(delim)
+
+	batcher := helpers.BuildBatcherFromArguments(c)
+	ext := helpers.BuildExtractorFromArguments(c, batcher)
+
+	writer := termrenderers.NewHeatmap(multiterm.New(), numRows, numCols)
+
+	writer.FixedMin = minFixed
+	writer.FixedMax = maxFixed
+	if minFixed || maxFixed {
+		writer.UpdateMinMax(minVal, maxVal)
+	}
+
+	helpers.RunAggregationLoop(ext, counter, func() {
+		writer.WriteTable(counter)
+		writer.WriteFooter(0, helpers.FWriteExtractorSummary(ext, counter.ParseErrors(),
+			fmt.Sprintf("(R: %v; C: %v)", color.Wrapi(color.Yellow, counter.RowCount()), color.Wrapi(color.BrightBlue, counter.ColumnCount()))))
+		writer.WriteFooter(1, batcher.StatusString())
+	})
+
+	return helpers.DetermineErrorState(batcher, ext, counter)
+}
+
+func heatmapCommand() *cli.Command {
+	return helpers.AdaptCommandForExtractor(cli.Command{
+		Name:      "heatmap",
+		Aliases:   []string{"heat"},
+		ShortName: "hm",
+		Usage:     "Create a 2D heatmap of extracted data",
+		Description: `Creates a dense 2D visual of extracted data.  Each character
+		represents a single data-point, and can create an alternative visualization to
+		a table.  Unicode and color support required for effective display`,
+		Action: heatmapFunction,
+		Flags: []cli.Flag{
+			cli.StringFlag{
+				Name:  "delim",
+				Usage: "Character to tabulate on. Use {$} helper by default",
+				Value: expressions.ArraySeparatorString,
+			},
+			cli.IntFlag{
+				Name:  "num,n,rows",
+				Usage: "Number of elements (rows) to display",
+				Value: 20,
+			},
+			cli.IntFlag{
+				Name:  "cols",
+				Usage: "Number of columns to display",
+				Value: multiterm.TermCols() - 15,
+			},
+			cli.Int64Flag{
+				Name:  "min",
+				Usage: "Sets the lower bounds of the heatmap (default: auto)",
+			},
+			cli.Int64Flag{
+				Name:  "max",
+				Usage: "Sets the upper bounds of the heatmap (default: auto)",
+			},
+		},
+	})
+}

cmd/heatmap_test.go

@@ -0,0 +1,9 @@
+package cmd
+
+import "testing"
+
+func TestHeatmap(t *testing.T) {
+	testCommandSet(t, heatmapCommand(),
+		`-m "(.+) (\d+)" -e "{$ {1} {2}}" testdata/graph.txt`,
+	)
+}

docs/images/README.md

@@ -10,6 +10,12 @@ terminalizer render -o temp.gif output.yml
 gifsicle -O3 --colors 128 -i temp.gif -o output.gif
 ```
 
+Note on environment; Make sure bashrc when terminalizer starts is set by changing `command:` in config yaml
+```bash
+export PS1="$ "
+export PATH="./:$PATH"
+```
+
 ### Recording
 
 ```bash
@@ -40,6 +46,10 @@ rare bars -s -m '\[(.+?)\].*" (\d+)' -e '{buckettime {1} year}' -e '{2}' access.
 
 rare table -m '\[(.+?)\].*" (\d+)' -e '{buckettime {1} year}' -e '{2}' access.log
 
+### Heatmap
+
+rare heatmap -m '\[(.+?)\].*" (\d+)' -e "{timeattr {time {1}} yearweek}" -e "{2}" access.log
+
 ### Analyze bytes sent, only looking at 200's
 
 rare analyze -m '(\d{3}) (\d+)' -e '{2}' -i '{neq {1} 200}' access.log

docs/index.md

@@ -16,7 +16,7 @@ Supports various CLI-based graphing and metric formats (filter (grep-like), hist
 
 ## Features
 
- * Multiple summary formats including: filter (like grep), histogram, bar graphs, and numerical analysis
+ * Multiple summary formats including: filter (like grep), histogram, bar graphs, tables, heatmaps, and numerical analysis
  * File glob expansions (eg `/var/log/*` or `/var/log/*/*.log`) and `-R`
  * Optional gzip decompression (with `-z`)
  * Following `-f` or re-open following `-F` (use `--poll` to poll, and `--tail` to tail)

docs/usage/aggregators.md

@@ -177,3 +177,36 @@ Rows: 223; Cols: 6
 ```
 
 ![Gif of table](../images/rare-table.gif)
+
+## Heatmap
+
+```sh
+rare help heatmap
+```
+
+### Summary
+
+Create a dense, color-coded, version of table-data by using cells to display
+the strength of a value. Can either use `\x00` or the `{$ a b}` helper.  First
+element is the column name, followed by the row name.
+
+### Example
+
+```bash
+$ rare heatmap -m '\[(.+?)\].*" (\d+)' \
+    -e "{timeattr {time {1}} yearweek}" -e "{2}" access.log
+
+        - 0    5 22,602    9 45,204
+     2019-34..2019-41..2019-50..2020-15..2020-23..2020-31...2020-9
+200  1111111111111111111111111111111111111111111111111111111-11111
+206  -------------------------------------------------------------
+301  -------------------------------------------------------------
+304  -------------------------------------------------------------
+400  -------------------------------------------------------------
+404  33516265914153253212111-1511-13-141-1412-132111--14-1-1-13211
+405  -------------------------------------------------------------
+408  -------------------------------------------------------------
+Matched: 1,035,666 / 1,035,666 (R: 8; C: 61)
+```
+
+![Gif of heatmap](../images/heatmap.gif)

pkg/aggregation/table.go

@@ -1,6 +1,7 @@
 package aggregation
 
 import (
+	"math"
 	"rare/pkg/stringSplitter"
 	"sort"
 	"strconv"
@@ -101,9 +102,11 @@ func (s *TableAggregator) OrderedColumns() []string {
 
 func (s *TableAggregator) OrderedColumnsByName() []string {
 	keys := s.Columns()
+
 	sort.Slice(keys, func(i, j int) bool {
 		return keys[i] < keys[j]
 	})
+
 	return keys
 }
 
@@ -146,6 +149,36 @@ func (s *TableAggregator) OrderedRowsByName() []*TableRow {
 	return rows
 }
 
+func (s *TableAggregator) ComputeMin() (ret int64) {
+	ret = math.MaxInt64
+	for _, r := range s.rows {
+		for colKey := range s.cols {
+			if val := r.cols[colKey]; val < ret {
+				ret = val
+			}
+		}
+	}
+	if ret == math.MaxInt64 {
+		return 0
+	}
+	return
+}
+
+func (s *TableAggregator) ComputeMax() (ret int64) {
+	ret = math.MinInt64
+	for _, r := range s.rows {
+		for colKey := range s.cols {
+			if val := r.cols[colKey]; val > ret {
+				ret = val
+			}
+		}
+	}
+	if ret == math.MinInt64 {
+		return 0
+	}
+	return
+}
+
 // ColTotals returns column oriented totals (Do not change!)
 func (s *TableAggregator) ColTotal(k string) int64 {
 	return s.cols[k]

pkg/aggregation/table_test.go

@@ -41,6 +41,10 @@ func TestSimpleTable(t *testing.T) {
 
 	// Totals
 	assert.Equal(t, int64(5), table.Sum())
+
+	// Minmax
+	assert.Equal(t, int64(0), table.ComputeMin())
+	assert.Equal(t, int64(3), table.ComputeMax())
 }
 
 func TestTableMultiIncrement(t *testing.T) {
@@ -65,4 +69,8 @@ func TestTableMultiIncrement(t *testing.T) {
 	assert.Equal(t, int64(5), table.ColTotal("b"))
 	assert.Equal(t, int64(1), table.ColTotal("a"))
 	assert.Equal(t, int64(6), table.Sum())
+
+	// Minmax
+	assert.Equal(t, int64(0), table.ComputeMin())
+	assert.Equal(t, int64(5), table.ComputeMax())
 }