POC: Generate jsonnet library from upstream schema

grafonnext-go/go.mod

@@ -0,0 +1,25 @@
+module github.com/grafana/grafonnet-lib/grafonnext-go
+
+go 1.19
+
+require (
+	cuelang.org/go v0.4.3
+	github.com/grafana/grafana v1.9.2-0.20221107230407-93589c483967
+	github.com/grafana/thema v0.0.0-20221107225215-00ad2949c7bc
+)
+
+require (
+	github.com/cockroachdb/apd/v2 v2.0.2 // indirect
+	github.com/emicklei/proto v1.10.0 // indirect
+	github.com/golang/glog v1.0.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/wire v0.5.0 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
+	github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b // indirect
+	github.com/yalue/merged_fs v1.2.2 // indirect
+	golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect
+	golang.org/x/text v0.3.8 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

grafonnext-go/go.sum

@@ -0,0 +1,60 @@
+cuelang.org/go v0.4.3 h1:W3oBBjDTm7+IZfCKZAmC8uDG0eYfJL4Pp/xbbCMKaVo=
+cuelang.org/go v0.4.3/go.mod h1:7805vR9H+VoBNdWFdI7jyDR3QLUPp4+naHfbcgp55HI=
+github.com/cockroachdb/apd/v2 v2.0.2 h1:weh8u7Cneje73dDh+2tEVLUvyBc89iwepWCD8b8034E=
+github.com/cockroachdb/apd/v2 v2.0.2/go.mod h1:DDxRlzC2lo3/vSlmSoS7JkqbbrARPuFOGr0B9pvN3Gw=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/emicklei/proto v1.10.0 h1:pDGyFRVV5RvV+nkBK9iy3q67FBy9Xa7vwrOTE+g5aGw=
+github.com/emicklei/proto v1.10.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
+github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8=
+github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
+github.com/grafana/grafana v1.9.2-0.20221107230407-93589c483967 h1:xaZRT+bch+reHBaekUzm2BApFBNkt4alBGlleWPpubQ=
+github.com/grafana/grafana v1.9.2-0.20221107230407-93589c483967/go.mod h1:qD8jQlBap35uu9oteuokbWbIEDrh1CINpxeXcLOg/Hc=
+github.com/grafana/thema v0.0.0-20221107225215-00ad2949c7bc h1:Icv777/PBaqhLmbSBSDaajDl424cbmh5ee77Du2rUFE=
+github.com/grafana/thema v0.0.0-20221107225215-00ad2949c7bc/go.mod h1:wnIJykzNiNVANl6g/Z4nkXxoMqaaH1LoG0IPNW++BEk=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk=
+github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
+github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
+github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de h1:D5x39vF5KCwKQaw+OC9ZPiLVHXz3UFw2+psEX+gYcto=
+github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de/go.mod h1:kJun4WP5gFuHZgRjZUWWuH1DTxCtxbHDOIJsudS8jzY=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b h1:zd/2RNzIRkoGGMjE+YIsZ85CnDIz672JK2F3Zl4vux4=
+github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b/go.mod h1:KjY0wibdYKc4DYkerHSbguaf3JeIPGhNJBp2BNiFH78=
+github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/yalue/merged_fs v1.2.2 h1:vXHTpJBluJryju7BBpytr3PDIkzsPMpiEknxVGPhN/I=
+github.com/yalue/merged_fs v1.2.2/go.mod h1:WqqchfVYQyclV2tnR7wtRhBddzBvLVR83Cjw9BKQw0M=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI=
+golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

grafonnext-go/mkschema.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"cuelang.org/go/cue/ast"
+	"github.com/grafana/grafana/pkg/cuectx"
+	"github.com/grafana/grafana/pkg/plugins/pfs/corelist"
+	"github.com/grafana/grafana/pkg/registry/corekind"
+	"github.com/grafana/thema/encoding/jsonschema"
+)
+
+// this is a meaningless hack, we have to work out versioning
+var grafanaVersion = "latest"
+
+var rt = cuectx.GrafanaThemaRuntime()
+
+func main() {
+	// All core kinds first
+	base := filepath.Join("..", "grafonnext", "schemas", grafanaVersion, "core")
+	die(os.MkdirAll(base, 0777))
+
+	for _, kind := range corekind.NewBase(nil).AllStructured() {
+		f, err := jsonschema.GenerateSchema(kind.Lineage().Latest())
+		if err != nil {
+			die(fmt.Errorf("%s: %w", kind.Name(), err))
+		}
+
+		b := marshalindent(f)
+		path := filepath.Join(base, kind.MachineName()+".json")
+		die(os.WriteFile(path, b, 0644))
+	}
+
+	// Now, all composable kinds from plugins. (Note that core grafana hasn't been refactored
+	// to use 'composable kind' naming for its various methods yet)
+	for _, ptree := range corelist.New(nil) {
+		for slot, lin := range ptree.RootPlugin().SlotImplementations() {
+			f, err := jsonschema.GenerateSchema(lin.Latest())
+			if err != nil {
+				// There's a bug in generating openapi from Thema with imports are loaded.
+				// Normally an edge case in Thema, but needed in Grafana. Commenting this out
+				// until bug is fixed and just dropping errors instead
+				// die(fmt.Errorf("%s: %w", ptree.RootPlugin().Meta().Id, err))
+				continue
+			}
+
+			b := marshalindent(f)
+			basecompo := filepath.Join("..", "grafonnext", "schemas", grafanaVersion, "composable", strings.ToLower(slot))
+			path := filepath.Join(basecompo, ptree.RootPlugin().Meta().Id+".json")
+			die(os.MkdirAll(basecompo, 0777))
+			die(os.WriteFile(path, b, 0644))
+		}
+	}
+}
+
+func die(err error) {
+	if err != nil {
+		fmt.Fprint(os.Stderr, err, "\n")
+		os.Exit(1)
+	}
+}
+
+func marshalindent(f *ast.File) []byte {
+	b, _ := rt.Context().BuildFile(f).MarshalJSON()
+	nb := new(bytes.Buffer)
+	die(json.Indent(nb, b, "", "  "))
+	return nb.Bytes()
+}

grafonnext/.gitignore

@@ -0,0 +1,2 @@
+jsonnetfile.lock.json
+vendor

grafonnext/Makefile

@@ -0,0 +1,11 @@
+ROOT:=$PWD
+
+schemas/%:
+	cd ../grafonnext-go && go run .
+#	sh bin/mkschema.sh $* $(abspath schemas)
+
+schemas.libsonnet:
+	sh bin/mkimport.sh
+
+static-%.libsonnet:
+	jsonnet -S -J vendor -A version='$*' -A render='static' main.libsonnet | jsonnetfmt - > static-$*.libsonnet

grafonnext/bin/mkimport.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+function gen() {
+    echo "{"
+    echo "// Generated with \`make schemas.libsonnet\`"
+    for VERSION in $(find schemas/* -maxdepth 0 -type d); do
+        V=$(basename $VERSION)
+        echo "'$V': ["
+        for FILE in $(find $VERSION/* -type f); do
+            echo "import '$FILE',"
+        done
+        echo "],"
+    done
+    echo "}"
+}
+
+gen | jsonnetfmt - > schemas.libsonnet

grafonnext/bin/mkschema.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+if [ "$#" != 2 ]; then
+  echo "Usage: $(basename "$0") VERSION OUTPUTDIR"
+  exit 1
+fi
+
+VERSION="$1"
+OUTPUTDIR="$2"
+
+mkdir -p "$OUTPUTDIR/$VERSION"
+
+TEMPDIR=$(mktemp -d)
+
+trap "rm -rf $TEMPDIR" EXIT
+
+git clone --branch "$VERSION" --depth 1 https://github.com/grafana/grafana.git "$TEMPDIR"
+cd "$TEMPDIR"
+
+cd pkg/coremodel/dashboard
+thema lineage gen jsonschema > "$OUTPUTDIR/$VERSION/dashboard.json"
+cd -
+
+# Made an assumption here, generating jsonschema for these don't work.
+#for FILE in $(find public -name models.cue); do
+#    DIR=$(dirname "$FILE")
+#    NAME=$(basename "$DIR")
+#    cd "$DIR"
+#    thema lineage gen jsonschema > "$OUTPUTDIR/$VERSION/$NAME.json"
+#    cd -
+#done

grafonnext/example.libsonnet

@@ -0,0 +1,8 @@
+local grafonnet = (import './main.libsonnet')('latest');
+local dashboard = grafonnet.dashboard;
+local panel = grafonnet.dashboard.panels.panel;
+
+dashboard.new('testDashboard')
++ dashboard.withPanels(
+  panel.new('testPanel', 'timeseries')
+)

grafonnext/jsonnetfile.json

@@ -0,0 +1,15 @@
+{
+  "version": 1,
+  "dependencies": [
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/Duologic/crdsonnet.git",
+          "subdir": "crdsonnet"
+        }
+      },
+      "version": "master"
+    }
+  ],
+  "legacyImports": true
+}

grafonnext/main.libsonnet

@@ -0,0 +1,43 @@
+local crdsonnet = import 'github.com/Duologic/crdsonnet/crdsonnet/main.libsonnet';
+local renderer = import 'github.com/Duologic/crdsonnet/crdsonnet/render.libsonnet';
+local schemasRaw = import 'schemas.libsonnet';
+
+function(version='latest', render='dynamic')
+
+  local schemas = {
+    [s.info.title]:
+      s
+      + (if 'PanelOptions' in s.components.schemas[s.info.title].properties
+         then { components+: { schemas+: {
+           [s.info.title]: { '$ref': '#/components/schemas/PanelOptions' },
+           PanelOptions: s.components.schemas[s.info.title].properties.PanelOptions,
+         } } }
+         else {})
+      + (if 'PanelLayout' in s.components.schemas[s.info.title].properties
+         then { components+: { schemas+: {
+           PanelLayout: s.components.schemas[s.info.title].properties.PanelLayout,
+         } } }
+         else {})
+    for s in schemasRaw[version]
+  };
+
+  (if render == 'dynamic'
+   then {}
+   else '// Generated with `make static-%s.libsonnet`\n' % version)
+  + std.foldl(
+    function(acc, k)
+      local schema = schemas[k];
+      local component = schema.components.schemas[k];
+      acc +
+      crdsonnet.fromOpenAPI(
+        k,
+        component,
+        schema,
+        render=render,
+      ),
+    std.objectFields(schemas),
+    renderer[render].nilvalue
+  )
+  + (if render == 'dynamic'
+     then (import 'veneer.libsonnet')
+     else "\n+ (import 'veneer.libsonnet')")

grafonnext/schemas.libsonnet

@@ -0,0 +1,11 @@
+{
+  // Generated with `make schemas.libsonnet`
+  latest: [
+    import 'schemas/latest/composable/panel/annolist.json',
+    import 'schemas/latest/composable/panel/text.json',
+    import 'schemas/latest/composable/panel/news.json',
+    import 'schemas/latest/composable/panel/dashlist.json',
+    import 'schemas/latest/core/dashboard.json',
+    import 'schemas/latest/core/playlist.json',
+  ],
+}

grafonnext/schemas/latest/composable/panel/annolist.json

@@ -0,0 +1,82 @@
+{
+  "openapi": "3.0.0",
+  "info": {
+    "title": "annolist",
+    "version": "0.0"
+  },
+  "paths": {},
+  "components": {
+    "schemas": {
+      "annolist": {
+        "type": "object",
+        "required": [
+          "PanelOptions"
+        ],
+        "properties": {
+          "PanelOptions": {
+            "type": "object",
+            "required": [
+              "onlyFromThisDashboard",
+              "onlyInTimeRange",
+              "tags",
+              "limit",
+              "showUser",
+              "showTime",
+              "showTags",
+              "navigateToPanel",
+              "navigateBefore",
+              "navigateAfter"
+            ],
+            "properties": {
+              "onlyFromThisDashboard": {
+                "type": "boolean",
+                "default": false
+              },
+              "onlyInTimeRange": {
+                "type": "boolean",
+                "default": false
+              },
+              "tags": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              },
+              "limit": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 4294967295,
+                "default": 10
+              },
+              "showUser": {
+                "type": "boolean",
+                "default": true
+              },
+              "showTime": {
+                "type": "boolean",
+                "default": true
+              },
+              "showTags": {
+                "type": "boolean",
+                "default": true
+              },
+              "navigateToPanel": {
+                "type": "boolean",
+                "default": true
+              },
+              "navigateBefore": {
+                "type": "string",
+                "default": "10m"
+              },
+              "navigateAfter": {
+                "type": "string",
+                "default": "10m"
+              }
+            }
+          }
+        },
+        "$schema": "http://json-schema.org/draft-04/schema#"
+      }
+    }
+  }
+}