Skip to content
This repository has been archived by the owner on Jun 26, 2023. It is now read-only.

POC: Generate jsonnet library from upstream schema #376

Closed
wants to merge 8 commits into from
Closed

POC: Generate jsonnet library from upstream schema #376

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
806fdce
poc: Generate jsonnet library from upstream schema
Duologic Nov 6, 2022
4ce2c86
provide both dynamic and static rendering
Duologic Nov 7, 2022
52b698f
use backticks
Duologic Nov 7, 2022
da54d8b
poc: Add Go-based generator of JSON Schema
Nov 8, 2022
95f721f
refactor: use go version of mkschema
Duologic Nov 9, 2022
0732695
fix example
Duologic Nov 9, 2022
8b5616b
make schemas.libsonnet
Duologic Nov 9, 2022
72533fb
fix: workaround bug in panel schemas
Duologic Nov 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions grafonnext/.gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
jsonnetfile.lock.json
vendor
10 changes: 10 additions & 0 deletions grafonnext/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
ROOT:=$PWD

schemas/%:
sh bin/mkschema.sh $* $(abspath schemas)

schemas.libsonnet:
sh bin/mkimport.sh

static-%.libsonnet:
jsonnet -S -J vendor -A version='$*' -A render='static' main.libsonnet | jsonnetfmt - > static-$*.libsonnet
19 changes: 19 additions & 0 deletions grafonnext/bin/mkimport.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'

function gen() {
echo "{"
echo "// Generated with \`make schemas.libsonnet\`"
for VERSION in $(find schemas/* -maxdepth 0 -type d); do
V=$(basename $VERSION)
echo "'$V': ["
for FILE in $(find $VERSION/* -maxdepth 0 -type f); do
echo "import '$FILE',"
done
echo "],"
done
echo "}"
}

gen | jsonnetfmt - > schemas.libsonnet
33 changes: 33 additions & 0 deletions grafonnext/bin/mkschema.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'

if [ "$#" != 2 ]; then
echo "Usage: $(basename "$0") VERSION OUTPUTDIR"
exit 1
fi

VERSION="$1"
OUTPUTDIR="$2"

mkdir -p "$OUTPUTDIR/$VERSION"

TEMPDIR=$(mktemp -d)

trap "rm -rf $TEMPDIR" EXIT

git clone --branch "$VERSION" --depth 1 https://github.com/grafana/grafana.git "$TEMPDIR"
cd "$TEMPDIR"

cd pkg/coremodel/dashboard
thema lineage gen jsonschema > "$OUTPUTDIR/$VERSION/dashboard.json"
cd -

# Made an assumption here, generating jsonschema for these don't work.
#for FILE in $(find public -name models.cue); do
# DIR=$(dirname "$FILE")
# NAME=$(basename "$DIR")
# cd "$DIR"
# thema lineage gen jsonschema > "$OUTPUTDIR/$VERSION/$NAME.json"
Duologic marked this conversation as resolved.
Show resolved Hide resolved
# cd -
#done
Duologic marked this conversation as resolved.
Show resolved Hide resolved
8 changes: 8 additions & 0 deletions grafonnext/example.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
local grafonnet = (import './main.libsonnet')('v9.2.3');
local dashboard = grafonnet.dashboard;
local panel = grafonnet.dashboard.panels.panel;

dashboard.new('testDashboard')
+ dashboard.withPanels(
panel.new('testPanel', 'timeseries')
)
25 changes: 25 additions & 0 deletions grafonnext/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
module github.com/grafana/grafonnet-lib/grafonnext

go 1.19

require (
cuelang.org/go v0.4.3
github.com/grafana/grafana v1.9.2-0.20221107230407-93589c483967
github.com/grafana/thema v0.0.0-20221107225215-00ad2949c7bc
)

require (
github.com/cockroachdb/apd/v2 v2.0.2 // indirect
github.com/emicklei/proto v1.10.0 // indirect
github.com/golang/glog v1.0.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/google/wire v0.5.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b // indirect
github.com/yalue/merged_fs v1.2.2 // indirect
golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect
golang.org/x/text v0.3.8 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
62 changes: 62 additions & 0 deletions grafonnext/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
cuelang.org/go v0.4.3 h1:W3oBBjDTm7+IZfCKZAmC8uDG0eYfJL4Pp/xbbCMKaVo=
cuelang.org/go v0.4.3/go.mod h1:7805vR9H+VoBNdWFdI7jyDR3QLUPp4+naHfbcgp55HI=
github.com/cockroachdb/apd/v2 v2.0.2 h1:weh8u7Cneje73dDh+2tEVLUvyBc89iwepWCD8b8034E=
github.com/cockroachdb/apd/v2 v2.0.2/go.mod h1:DDxRlzC2lo3/vSlmSoS7JkqbbrARPuFOGr0B9pvN3Gw=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/emicklei/proto v1.10.0 h1:pDGyFRVV5RvV+nkBK9iy3q67FBy9Xa7vwrOTE+g5aGw=
github.com/emicklei/proto v1.10.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8=
github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
github.com/grafana/grafana v1.9.2-0.20221107225804-85217355b416 h1:kuVO9kBoK44gUuIW3KtAGK6H6lnfRxDtLxfix3LgIAk=
github.com/grafana/grafana v1.9.2-0.20221107225804-85217355b416/go.mod h1:qD8jQlBap35uu9oteuokbWbIEDrh1CINpxeXcLOg/Hc=
github.com/grafana/grafana v1.9.2-0.20221107230407-93589c483967 h1:xaZRT+bch+reHBaekUzm2BApFBNkt4alBGlleWPpubQ=
github.com/grafana/grafana v1.9.2-0.20221107230407-93589c483967/go.mod h1:qD8jQlBap35uu9oteuokbWbIEDrh1CINpxeXcLOg/Hc=
github.com/grafana/thema v0.0.0-20221107225215-00ad2949c7bc h1:Icv777/PBaqhLmbSBSDaajDl424cbmh5ee77Du2rUFE=
github.com/grafana/thema v0.0.0-20221107225215-00ad2949c7bc/go.mod h1:wnIJykzNiNVANl6g/Z4nkXxoMqaaH1LoG0IPNW++BEk=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk=
github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de h1:D5x39vF5KCwKQaw+OC9ZPiLVHXz3UFw2+psEX+gYcto=
github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de/go.mod h1:kJun4WP5gFuHZgRjZUWWuH1DTxCtxbHDOIJsudS8jzY=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b h1:zd/2RNzIRkoGGMjE+YIsZ85CnDIz672JK2F3Zl4vux4=
github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b/go.mod h1:KjY0wibdYKc4DYkerHSbguaf3JeIPGhNJBp2BNiFH78=
github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
github.com/yalue/merged_fs v1.2.2 h1:vXHTpJBluJryju7BBpytr3PDIkzsPMpiEknxVGPhN/I=
github.com/yalue/merged_fs v1.2.2/go.mod h1:WqqchfVYQyclV2tnR7wtRhBddzBvLVR83Cjw9BKQw0M=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI=
golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
15 changes: 15 additions & 0 deletions grafonnext/jsonnetfile.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{
"version": 1,
"dependencies": [
{
"source": {
"git": {
"remote": "https://github.com/Duologic/crdsonnet.git",
"subdir": "crdsonnet"
}
},
"version": "master"
}
],
"legacyImports": true
}
24 changes: 24 additions & 0 deletions grafonnext/main.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
local crdsonnet = import 'github.com/Duologic/crdsonnet/crdsonnet/main.libsonnet';
local render = import 'github.com/Duologic/crdsonnet/crdsonnet/render.libsonnet';
local schemas = import 'schemas.libsonnet';

function(version='v9.2.3', render='dynamic')

local schema = {
[s.info.title]: s
for s in schemas[version]
}.dashboard;
local component = schema.components.schemas.dashboard;

(if render == 'dynamic'
then {}
else '// Generated with `make static-%s.libsonnet`\n' % version)
+ crdsonnet.fromOpenAPI(
'dashboard',
component,
schema,
render=render,
)
+ (if render == 'dynamic'
then (import 'veneer.libsonnet')
else "\n+ (import 'veneer.libsonnet')")
73 changes: 73 additions & 0 deletions grafonnext/mkschema.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package main

import (
"bytes"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"

"cuelang.org/go/cue/ast"
"github.com/grafana/grafana/pkg/cuectx"
"github.com/grafana/grafana/pkg/plugins/pfs/corelist"
"github.com/grafana/grafana/pkg/registry/corekind"
"github.com/grafana/thema/encoding/jsonschema"
)

// this is a meaningless hack, we have to work out versioning
var grafanaVersion = "latest"
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is just very clearly not what you're looking for here. And it's equally clear to me that we do not currently have sufficient information available in Go scope to organize these outputs by Grafana release version.

That said, working on this gave me a thought about how we might be able to make that information available in a tidy way that would be useful here and elsewhere. Might be a good thing for @malcolmholmes to look at, as it straddles some known overlaps with release engineering.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like this will be covered by grafana/grafana#58498, right?

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Exactly!


var rt = cuectx.GrafanaThemaRuntime()

func main() {
// All core kinds first
base := filepath.Join("schemas", grafanaVersion, "core")
die(os.MkdirAll(base, 0777))

for _, kind := range corekind.NewBase(nil).AllStructured() {
Copy link

@sdboyer sdboyer Nov 8, 2022 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Probably the main thing that using the kind as represented in Go offers is all the cue metadata/DSL we'll be wrapping around the kind declarations. In a Go context, these structs will be kept in sync with what's in those cue framework definitions, and are easy to access. It will still probably be possible with generic CLI tooling - use thema to grab out lineages from known subpaths, and then cue to extract certain known metadata from other subpaths, emit it as JSON, etc. But i suspect that will get increasingly unwieldy compared to just using the system designed to make this all easy to access from Go, in an external repo, as we're doing here.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The main area of interest right now with this is probably maturity flags, btw - it seems wise to indicate something about the maturity levels and corresponding guarantees that come with them for each schema.

My stance on grok is that i'm not willing to put it out at all until we have clear comms in that regard. I understand how there are different considerations here, where there were never guarantees in the first place. So probably fine to not consider that part of MVP here.

f, err := jsonschema.GenerateSchema(kind.Lineage().Latest())
if err != nil {
die(fmt.Errorf("%s: %w", kind.Name(), err))
}

b := marshalindent(f)
path := filepath.Join(base, kind.MachineName()+".json")
die(os.WriteFile(path, b, 0644))
}

// Now, all composable kinds from plugins. (Note that core grafana hasn't been refactored
// to use 'composable kind' naming for its various methods yet)
for _, ptree := range corelist.New(nil) {
for slot, lin := range ptree.RootPlugin().SlotImplementations() {
f, err := jsonschema.GenerateSchema(lin.Latest())
if err != nil {
// There's a bug in generating openapi from Thema with imports are loaded.
// Normally an edge case in Thema, but needed in Grafana. Commenting this out
// until bug is fixed and just dropping errors instead
// die(fmt.Errorf("%s: %w", ptree.RootPlugin().Meta().Id, err))
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this is uncommented, then we get errors like the following:

barchart: package "github.com/grafana/grafana/packages/grafana-schema/src/schema" imported but not defined in  (and 5 more errors)

This happens because of a fugly hack i have to do in Thema in order to be able to generate OpenAPI at all, because the OpenAPI encoder uses the old, deprecated cue.Instance. Basically, that func writes out a text version of the lineage and recompiles it through a backdoor - but when recompiling, external imports can't be sanely satisfied.

They fixed the API in the recent alpha release, so i no longer have to export, but...well, there's another problem, and i probably just need to submit a patch upstream. Hoping to do that this week - this is also a prerequisite for grok, so it needs to get done regardless.

continue
}

b := marshalindent(f)
basecompo := filepath.Join("schemas", grafanaVersion, "composable", strings.ToLower(slot))
path := filepath.Join(basecompo, ptree.RootPlugin().Meta().Id+".json")
die(os.MkdirAll(basecompo, 0777))
die(os.WriteFile(path, b, 0644))
}
}
}

func die(err error) {
if err != nil {
fmt.Fprint(os.Stderr, err, "\n")
os.Exit(1)
}
}

func marshalindent(f *ast.File) []byte {
b, _ := rt.Context().BuildFile(f).MarshalJSON()
nb := new(bytes.Buffer)
die(json.Indent(nb, b, "", " "))
return nb.Bytes()
}
6 changes: 6 additions & 0 deletions grafonnext/schemas.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
// Generated with `make schemas.libsonnet`
'v9.2.3': [
import 'schemas/v9.2.3/dashboard.json',
],
}
82 changes: 82 additions & 0 deletions grafonnext/schemas/latest/composable/panel/annolist.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
{
"openapi": "3.0.0",
"info": {
"title": "annolist",
"version": "0.0"
},
"paths": {},
"components": {
"schemas": {
"annolist": {
"type": "object",
"required": [
"PanelOptions"
],
"properties": {
"PanelOptions": {
"type": "object",
"required": [
"onlyFromThisDashboard",
"onlyInTimeRange",
"tags",
"limit",
"showUser",
"showTime",
"showTags",
"navigateToPanel",
"navigateBefore",
"navigateAfter"
],
"properties": {
"onlyFromThisDashboard": {
"type": "boolean",
"default": false
},
"onlyInTimeRange": {
"type": "boolean",
"default": false
},
"tags": {
"type": "array",
"items": {
"type": "string"
}
},
"limit": {
"type": "integer",
"minimum": 0,
"maximum": 4294967295,
"default": 10
},
"showUser": {
"type": "boolean",
"default": true
},
"showTime": {
"type": "boolean",
"default": true
},
"showTags": {
"type": "boolean",
"default": true
},
"navigateToPanel": {
"type": "boolean",
"default": true
},
"navigateBefore": {
"type": "string",
"default": "10m"
},
"navigateAfter": {
"type": "string",
"default": "10m"
}
}
}
},
"$schema": "http://json-schema.org/draft-04/schema#"
}
}
}
}