cmd/streamdiff: new command that diffs over a json stream

README.md

@@ -1,12 +1,16 @@
-Ripta's collection of tools
+rt: Ripta's collection of tools
 
-Tools:
+Said tools:
 
 * [enc](#enc) to encode and decode strings
 * [grpcto](#grpcto) to frame and unframe gRPC messages
 * [place](#place) for macOS Location Services
+* [streamdiff](#streamdiff) to help you pick out field changes off a stream of JSON
 * [toto](#toto) to inspect some protobuf messages
 * [uni](#uni) for unicode utils
+* [yfmt](#yfmt) to reindent YAML while preserving comments
+
+Pull requests welcome, though you should probably check first before sinking any time.
 
 `enc`
 ----
@@ -76,6 +80,66 @@ Last observed: 2022-02-02T21:24:40-08:00
 
 or as JSON by giving `-j` or `--json`.
 
+`streamdiff`
+------------
+
+Helps you pick out field changes off a stream of JSON.
+
+```
+go install github.com/ripta/rt/cmd/streamdiff@latest
+```
+
+It's technically usable  on any stream as long as the format is one JSON per
+line.
+
+It's convenient for viewing Kubernetes resource changes over time.
+
+For example, you can start a watch (`-w`) on pods (`kubectl get pods`) and
+pipe it to streamdiff. Most fields won't be printed, except when they change.
+Consider this output:
+
+```
+❯ kubectl get pods -o json -w | streamdiff
+T+23s Pod:pomerium-cache-6c9f84b747-cr2rx
+  (1/2): spec.nodeName \ -> gke-vqjp-preemptible-065-38c45f41-wtnb
+  (2/2): status.conditions \ -> [map[lastProbeTime:<nil> lastTransitionTime:2023-06-22T06:27:43Z status:True type:PodScheduled]]
+
+T+24s Pod:pomerium-cache-6c9f84b747-cr2rx
+  (1/6): status.conditions.0 \ -> map[lastProbeTime:<nil> lastTransitionTime:2023-06-22T06:27:43Z status:True type:Initialized]
+  (2/6): status.conditions.1 \ -> map[lastProbeTime:<nil> lastTransitionTime:2023-06-22T06:27:43Z message:containers with unready status: [cache] reason:ContainersNotReady status:False type:Ready]
+  (3/6): status.conditions.2 \ -> map[lastProbeTime:<nil> lastTransitionTime:2023-06-22T06:27:43Z message:containers with unready status: [cache] reason:ContainersNotReady status:False type:ContainersReady]
+  (4/6): status.startTime \ -> 2023-06-22T06:27:43Z
+  (5/6): status.containerStatuses \ -> [map[image:us.gcr.io/dc-02/gke-vqjp/pomerium-cache:v1.0.23.1390 imageID: lastState:map[] name:cache ready:false restartCount:0 started:false state:map[waiting:map[reason:ContainerCreating]]]]
+  (6/6): status.hostIP \ -> 10.52.0.34
+
+T+26s Pod:pomerium-cache-6c9f84b747-cr2rx
+  (1/8): status.containerStatuses.0.ready false -> true
+  (2/8): status.containerStatuses.0.started false -> true
+  (3/8): status.containerStatuses.0.state.waiting map[reason:ContainerCreating] -> \
+  (4/8): status.containerStatuses.0.state.running \ -> map[startedAt:2023-06-22T06:27:46Z]
+  (5/8): status.containerStatuses.0.containerID \ -> containerd://293972feb5b498c80a585137299990c77f44ea46d6236432aba08e72108c35dc
+  (6/8): status.phase Pending -> Running
+  (7/8): status.podIP \ -> 10.53.1.92
+  (8/8): status.podIPs \ -> [map[ip:10.53.1.92]]
+```
+
+While there is still some noise, it clearly shows when the pod was assigned to
+a node, when the pod finished initializing, and when it changed phases from
+Pending to Running.
+
+In addition to a running log (as above), you can also run `streamdiff -i`,
+which updates status on the same line instead of printing a new line for
+every resource update. YMMV.
+
+```
+❯ kubectl get nodes -o json -w | streamdiff -i
+\ Node:gke-vqjp-ondemand-370-504f82ce-r0d8	status.conditions.0.{type: FrequentContainerdRestart; status: True -> False} 
+\ Node:gke-vqjp-preemptible-065-38c45f41-kvjd	status.conditions.0.lastHeartbeatTime: 2023-06-22T06:44:18Z -> 2023-06-22T06:49:19Z
+| Node:gke-vqjp-preemptible-065-38c45f41-pklf	status.conditions.0.lastHeartbeatTime: 2023-06-22T06:44:15Z -> 2023-06-22T06:49:16Z
+/ Node:gke-vqjp-preemptible-065-38c45f41-wtnb	status.conditions.0.lastHeartbeatTime: 2023-06-22T06:45:05Z -> 2023-06-22T06:50:11Z
+```
+
+
 `toto`
 ------
 
@@ -104,6 +168,8 @@ files directly in go (or at least, I wasn't able to).
 `uni`
 -----
 
+Unicode-related stuff.
+
 ```
 go install github.com/ripta/rt/cmd/uni@latest
 ```
@@ -166,3 +232,32 @@ U+1101	ᄁ	HANGUL CHOSEONG SSANGKIYEOK
 U+116E	ᅮ	HANGUL JUNGSEONG U
 U+000A	"\n"	<control>
 ```
+
+
+`yfmt`
+------
+
+Reindent YAML while preserving comments.
+
+```
+go install github.com/ripta/rt/cmd/yfmt@latest
+```
+
+This tool treats comments as nodes and therefore will _not_ preserve comment
+indentation. For example:
+
+```
+❯ cat in.yaml
+# does this work?
+foo:
+   - 123   # I hope
+           # maybe
+   - 456
+
+❯ yfmt < in.yaml
+# does this work?
+foo:
+  - 123 # I hope
+  # maybe
+  - 456
+```

cloudbuild.yaml

@@ -1,5 +1,5 @@
 steps:
-  - name: 'golang:1.19-bullseye'
+  - name: 'golang:1.20-bookworm'
     args:
       - bash
       - -c

go.mod

@@ -1,6 +1,6 @@
 module github.com/ripta/rt
 
-go 1.19
+go 1.20
 
 require (
 	github.com/containerd/console v1.0.3
@@ -9,17 +9,20 @@ require (
 	github.com/mr-tron/base58 v1.2.0
 	github.com/r3labs/diff/v3 v3.0.1
 	github.com/spf13/cobra v1.7.0
+	github.com/stretchr/testify v1.8.4
 	github.com/thediveo/enumflag/v2 v2.0.4
 	golang.org/x/text v0.10.0
 	google.golang.org/protobuf v1.30.0
 	gopkg.in/yaml.v3 v3.0.1
-	k8s.io/apimachinery v0.27.3
 )
 
 require (
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stoewer/go-strcase v1.2.0 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect

go.sum

@@ -1,10 +1,14 @@
 github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18=
 github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw=
 github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
@@ -13,8 +17,10 @@ github.com/google/cel-go v0.16.0/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulN
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
+github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY=
 github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
@@ -38,7 +44,8 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/thediveo/enumflag/v2 v2.0.4 h1:CPez2ZDJMkJ0iPiueJ6/vwsFeFy+w5kIJNFwxKPSUGo=
 github.com/thediveo/enumflag/v2 v2.0.4/go.mod h1:K5VGebAdhHGZyVprL7WEnEJ3CA16YzWhDH2ERwddA0I=
 github.com/thediveo/success v1.0.1 h1:NVwUOwKUwaN8szjkJ+vsiM2L3sNBFscldoDJ2g2tAPg=
@@ -49,6 +56,7 @@ github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
@@ -68,5 +76,3 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-k8s.io/apimachinery v0.27.3 h1:Ubye8oBufD04l9QnNtW05idcOe9Z3GQN8+7PqmuVcUM=
-k8s.io/apimachinery v0.27.3/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E=

pkg/streamdiff/decoder/splitter.go

@@ -0,0 +1,71 @@
+package decoder
+
+import (
+	"reflect"
+	"strings"
+)
+
+type SplitterFunc func(any) ([]any, bool)
+
+func KubernetesListSplitter(obj any) ([]any, bool) {
+	rv := reflect.ValueOf(obj)
+	if !rv.IsValid() {
+		return nil, false
+	}
+	if rv.IsNil() {
+		return nil, false
+	}
+	if rv.Kind() != reflect.Map {
+		return nil, false
+	}
+
+	rvKind, ok := traversePath(rv, []any{"kind"})
+	if !ok {
+		return nil, false
+	}
+
+	valKind, ok := rvKind.Interface().(string)
+	if !ok {
+		return nil, false
+	}
+
+	// fmt.Printf("YYY: %s\n", valKind)
+	if !strings.HasSuffix(valKind, "List") {
+		return nil, false
+	}
+
+	return GenerateSplitter("items")(obj)
+}
+
+func GenerateSplitter(paths ...any) func(obj any) ([]any, bool) {
+	return func(obj any) ([]any, bool) {
+		rv := reflect.ValueOf(obj)
+		if !rv.IsValid() {
+			return nil, false
+		}
+		if rv.IsNil() {
+			return nil, false
+		}
+		if rv.Kind() != reflect.Map {
+			return nil, false
+		}
+
+		rvItems, ok := traversePath(rv, paths)
+		if !ok {
+			return nil, false
+		}
+
+		//valItems, ok := rvItems.Interface().([]map[string]any)
+		//if ok {
+		//	anyItems := []any{}
+		//	for i := range valItems {
+		//		anyItems = append(anyItems, valItems[i])
+		//	}
+		//
+		//	return anyItems, true
+		//}
+
+		anyItems, ok := rvItems.Interface().([]any)
+		return anyItems, ok
+	}
+}

pkg/streamdiff/decoder/splitter_test.go

@@ -0,0 +1,94 @@
+package decoder
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type kubernetesListSplitterTest struct {
+	label         string
+	input         any
+	expectedFail  bool
+	expectedSplit []any
+}
+
+var kubernetesListSplitterTests = []kubernetesListSplitterTest{
+	{
+		label:         "empty_must_not_panic",
+		input:         nil,
+		expectedFail:  true,
+		expectedSplit: nil,
+	},
+	{
+		label:         "nothing_to_split",
+		input:         map[string]any{},
+		expectedFail:  true,
+		expectedSplit: nil,
+	},
+	{
+		label: "non_list_resource",
+		input: map[string]any{
+			"apiVersion": "v1",
+			"kind":       "Node",
+			"metadata": map[string]any{
+				"name": "foo-bar",
+			},
+		},
+		expectedFail:  true,
+		expectedSplit: nil,
+	},
+	{
+		label: "list_resource",
+		input: map[string]any{
+			"apiVersion": "v1",
+			"kind":       "NodeList",
+			"items": []any{
+				map[string]any{
+					"apiVersion": "v1",
+					"kind":       "Node",
+					"metadata": map[string]any{
+						"name": "foo",
+					},
+				},
+				map[string]any{
+					"apiVersion": "v1",
+					"kind":       "Node",
+					"metadata": map[string]any{
+						"name": "bar",
+					},
+				},
+			},
+		},
+		expectedFail: false,
+		expectedSplit: []any{
+			map[string]any{
+				"apiVersion": "v1",
+				"kind":       "Node",
+				"metadata": map[string]any{
+					"name": "foo",
+				},
+			},
+			map[string]any{
+				"apiVersion": "v1",
+				"kind":       "Node",
+				"metadata": map[string]any{
+					"name": "bar",
+				},
+			},
+		},
+	},
+}
+
+func TestKubernetesListSplitter(t *testing.T) {
+	for i := range kubernetesListSplitterTests {
+		test := kubernetesListSplitterTests[i]
+		t.Run(test.label, func(t *testing.T) {
+			actual, ok := KubernetesListSplitter(test.input)
+			assert.Equal(t, test.expectedFail, !ok)
+			if ok {
+				assert.Equal(t, test.expectedSplit, actual)
+			}
+		})
+	}
+}