Heatmap

README.md

@@ -18,7 +18,7 @@ See [rare.zdyn.net](https://rare.zdyn.net) or the [docs/ folder](docs/) for the
 
 ## Features
 
- * Multiple summary formats including: filter (like grep), histogram, bar graphs, and numerical analysis
+ * Multiple summary formats including: filter (like grep), histogram, bar graphs, tables, heatmaps, and numerical analysis
  * File glob expansions (eg `/var/log/*` or `/var/log/*/*.log`) and `-R`
  * Optional gzip decompression (with `-z`)
  * Following `-f` or re-open following `-F` (use `--poll` to poll, and `--tail` to tail)
@@ -37,6 +37,7 @@ Output formats include:
 * `filter` is grep-like, in that each line will be processed and the extracted key will be output directly to stdout
 * `histogram` will count instances of the extracted key
 * `table` will count the key in 2 dimensions
+* `heatmap` will display a color-coded version of the strength of a cell in a dense format
 * `bargraph` will create either a stacked or non-stacked bargraph based on 2 dimensions
 * `analyze` will use the key as a numeric value and compute mean/median/mode/stddev/percentiles
 

cmd/commands.go

@@ -5,6 +5,7 @@ import "github.com/urfave/cli"
 var commands []cli.Command = []cli.Command{
 	*filterCommand(),
 	*histogramCommand(),
+	*heatmapCommand(),
 	*bargraphCommand(),
 	*analyzeCommand(),
 	*tabulateCommand(),

cmd/heatmap.go

@@ -0,0 +1,67 @@
+package cmd
+
+import (
+	"fmt"
+	"rare/cmd/helpers"
+	"rare/pkg/aggregation"
+	"rare/pkg/color"
+	"rare/pkg/expressions"
+	"rare/pkg/multiterm"
+	"rare/pkg/multiterm/termrenderers"
+
+	"github.com/urfave/cli"
+)
+
+func heatmapFunction(c *cli.Context) error {
+	var (
+		delim   = c.String("delim")
+		numRows = c.Int("num")
+		numCols = c.Int("cols")
+	)
+
+	counter := aggregation.NewTable(delim)
+
+	batcher := helpers.BuildBatcherFromArguments(c)
+	ext := helpers.BuildExtractorFromArguments(c, batcher)
+
+	writer := termrenderers.NewHeatmap(multiterm.New(), numRows, numCols)
+
+	helpers.RunAggregationLoop(ext, counter, func() {
+		writer.WriteTable(counter)
+		writer.WriteFooter(0, helpers.FWriteExtractorSummary(ext, counter.ParseErrors(),
+			fmt.Sprintf("(R: %v; C: %v)", color.Wrapi(color.Yellow, counter.RowCount()), color.Wrapi(color.BrightBlue, counter.ColumnCount()))))
+		writer.WriteFooter(1, batcher.StatusString())
+	})
+
+	return helpers.DetermineErrorState(batcher, ext, nil)
+}
+
+func heatmapCommand() *cli.Command {
+	return helpers.AdaptCommandForExtractor(cli.Command{
+		Name:      "heatmap",
+		Aliases:   []string{"heat"},
+		ShortName: "hm",
+		Usage:     "Create a 2D heatmap of extracted data",
+		Description: `Creates a dense 2D visual of extracted data.  Each character
+		represents a single data-point, and can create an alternative visualization to
+		a table.  Unicode and color support required for effective display`,
+		Action: heatmapFunction,
+		Flags: []cli.Flag{
+			cli.StringFlag{
+				Name:  "delim",
+				Usage: "Character to tabulate on. Use {$} helper by default",
+				Value: expressions.ArraySeparatorString,
+			},
+			cli.IntFlag{
+				Name:  "num,n,rows",
+				Usage: "Number of elements (rows) to display",
+				Value: 20,
+			},
+			cli.IntFlag{
+				Name:  "cols",
+				Usage: "Number of columns to display",
+				Value: multiterm.TermCols() - 15,
+			},
+		},
+	})
+}

cmd/heatmap_test.go

@@ -0,0 +1,9 @@
+package cmd
+
+import "testing"
+
+func TestHeatmap(t *testing.T) {
+	testCommandSet(t, heatmapCommand(),
+		`-m "(.+) (\d+)" -e "{$ {1} {2}}" testdata/graph.txt`,
+	)
+}

docs/images/README.md

@@ -10,6 +10,12 @@ terminalizer render -o temp.gif output.yml
 gifsicle -O3 --colors 128 -i temp.gif -o output.gif
 ```
 
+Note on environment; Make sure bashrc when terminalizer starts is set by changing `command:` in config yaml
+```bash
+export PS1="$ "
+export PATH="./:$PATH"
+```
+
 ### Recording
 
 ```bash
@@ -40,6 +46,10 @@ rare bars -s -m '\[(.+?)\].*" (\d+)' -e '{buckettime {1} year}' -e '{2}' access.
 
 rare table -m '\[(.+?)\].*" (\d+)' -e '{buckettime {1} year}' -e '{2}' access.log
 
+### Heatmap
+
+rare heatmap -m '\[(.+?)\].*" (\d+)' -e "{timeattr {time {1}} yearweek}" -e "{2}" access.log
+
 ### Analyze bytes sent, only looking at 200's
 
 rare analyze -m '(\d{3}) (\d+)' -e '{2}' -i '{neq {1} 200}' access.log

docs/index.md

@@ -16,7 +16,7 @@ Supports various CLI-based graphing and metric formats (filter (grep-like), hist
 
 ## Features
 
- * Multiple summary formats including: filter (like grep), histogram, bar graphs, and numerical analysis
+ * Multiple summary formats including: filter (like grep), histogram, bar graphs, tables, heatmaps, and numerical analysis
  * File glob expansions (eg `/var/log/*` or `/var/log/*/*.log`) and `-R`
  * Optional gzip decompression (with `-z`)
  * Following `-f` or re-open following `-F` (use `--poll` to poll, and `--tail` to tail)

docs/usage/aggregators.md

@@ -177,3 +177,24 @@ Rows: 223; Cols: 6
 ```
 
 ![Gif of table](../images/rare-table.gif)
+
+## Heatmap
+
+```sh
+rare help heatmap
+```
+
+### Summary
+
+Create a dense, color-coded, version of table-data by using cells to display
+the strength of a value. Can either use `\x00` or the `{$ a b}` helper.  First
+element is the column name, followed by the row name.
+
+### Example
+
+```bash
+$ rare heatmap -m '\[(.+?)\].*" (\d+)' \
+    -e "{timeattr {time {1}} yearweek}" -e "{2}" access.log
+```
+
+![Gif of heatmap](../images/heatmap.gif)

pkg/aggregation/table.go

@@ -13,16 +13,19 @@ type TableRow struct {
 }
 
 type TableAggregator struct {
-	delim  string
-	errors uint64
-	rows   map[string]*TableRow
-	cols   map[string]int64 // Columns that track totals
+	delim    string
+	errors   uint64
+	min, max int64
+	rows     map[string]*TableRow
+	cols     map[string]int64 // Columns that track totals
 }
 
 func NewTable(delim string) *TableAggregator {
 	return &TableAggregator{
 		delim:  delim,
 		errors: 0,
+		min:    0,
+		max:    0,
 		rows:   make(map[string]*TableRow),
 		cols:   make(map[string]int64),
 	}
@@ -67,7 +70,15 @@ func (s *TableAggregator) SampleItem(colKey, rowKey string, inc int64) {
 		s.rows[rowKey] = row
 	}
 
-	row.cols[colKey] += inc
+	curr := row.cols[colKey]
+	curr += inc
+	if curr > s.max {
+		s.max = curr
+	}
+	if curr < s.min {
+		s.min = curr
+	}
+	row.cols[colKey] = curr
 	row.sum += inc
 }
 
@@ -101,9 +112,11 @@ func (s *TableAggregator) OrderedColumns() []string {
 
 func (s *TableAggregator) OrderedColumnsByName() []string {
 	keys := s.Columns()
+
 	sort.Slice(keys, func(i, j int) bool {
 		return keys[i] < keys[j]
 	})
+
 	return keys
 }
 
@@ -146,6 +159,14 @@ func (s *TableAggregator) OrderedRowsByName() []*TableRow {
 	return rows
 }
 
+func (s *TableAggregator) Min() int64 {
+	return s.min
+}
+
+func (s *TableAggregator) Max() int64 {
+	return s.max
+}
+
 // ColTotals returns column oriented totals (Do not change!)
 func (s *TableAggregator) ColTotal(k string) int64 {
 	return s.cols[k]

pkg/aggregation/table_test.go

@@ -41,6 +41,10 @@ func TestSimpleTable(t *testing.T) {
 
 	// Totals
 	assert.Equal(t, int64(5), table.Sum())
+
+	// Minmax
+	assert.Equal(t, int64(0), table.Min())
+	assert.Equal(t, int64(3), table.Max())
 }
 
 func TestTableMultiIncrement(t *testing.T) {
@@ -65,4 +69,8 @@ func TestTableMultiIncrement(t *testing.T) {
 	assert.Equal(t, int64(5), table.ColTotal("b"))
 	assert.Equal(t, int64(1), table.ColTotal("a"))
 	assert.Equal(t, int64(6), table.Sum())
+
+	// Minmax
+	assert.Equal(t, int64(0), table.Min())
+	assert.Equal(t, int64(6), table.Max())
 }