Kindsys: Introduce Kind framework

.github/CODEOWNERS

@@ -218,7 +218,15 @@ lerna.json @grafana/frontend-ops
 # Grafana Partnerships Team
 /pkg/infra/httpclient/httpclientprovider/sigv4_middleware.go @grafana/grafana-partnerships-team
 
-# Schema framework and code generation
+# Kind system and code generation
+embed.go @grafana/grafana-as-code
+/kinds/ @grafana/grafana-as-code
 /pkg/codegen @grafana/grafana-as-code
-/pkg/framework/coremodel @grafana/grafana-as-code
+/pkg/kindsys @grafana/grafana-as-code
+/pkg/kinds/*/*_gen.go @grafana/grafana-as-code
+/pkg/registry/corekind @grafana/grafana-as-code
 /public/app/plugins/*gen.go @grafana/grafana-as-code
+
+# Specific core kinds
+/kinds/raw/ @grafana/grafana-edge-squad
+/kinds/structured/dashboard @grafana/dashboards-squad

.gitignore

@@ -170,14 +170,8 @@ compilation-stats.json
 # auto generated frontend docs
 /docs/sources/packages_api
 
-# auto generated Go files
-*_gen.go
-!pkg/services/featuremgmt/toggles_gen.go
-!pkg/coremodel/**/*_gen.go
-!pkg/framework/**/*_gen.go
-!pkg/plugins/pfs/**/*_gen.go
-!public/app/plugins/**/*_gen.go
-!pkg/services/publicdashboards/*_gen.go
+# wire generated files
+**/wire_gen.go
 
 # Auto-generated internationalization files
 public/locales/_build/

LICENSING.md

@@ -17,10 +17,11 @@ packages/grafana-toolkit/
 packages/grafana-ui/
 packages/jaeger-ui-components/
 packaging/
-pkg/coremodel/
-pkg/framework/coremodel/
+kinds/
+pkg/kinds/
+pkg/kindsys/
+pkg/registry/corekind/
 grafana-mixin/
-cue/
 public/app/plugins/datasource/tempo
 public/img/icons/solid/
 public/img/icons/unicons/

Makefile

@@ -66,6 +66,7 @@ openapi3-gen: swagger-api-spec ## Generates OpenApi 3 specs from the Swagger 2 a
 ##@ Building
 gen-cue: ## Do all CUE/Thema code generation
 	@echo "generate code from .cue files"
+	go generate ./kinds/gen.go
 	go generate ./pkg/framework/coremodel
 	go generate ./public/app/plugins
 

embed.go

@@ -6,5 +6,5 @@ import (
 
 // CueSchemaFS embeds all schema-related CUE files in the Grafana project.
 //
-//go:embed cue.mod/module.cue packages/grafana-schema/src/schema/*.cue public/app/plugins/*/*/*.cue public/app/plugins/*/*/plugin.json pkg/framework/coremodel/*.cue
+//go:embed cue.mod/module.cue kinds/*/*.cue kinds/*/*/*.cue packages/grafana-schema/src/schema/*.cue public/app/plugins/*/*/*.cue public/app/plugins/*/*/plugin.json pkg/framework/coremodel/*.cue pkg/kindsys/*.cue
 var CueSchemaFS embed.FS

go.mod

@@ -61,7 +61,7 @@ require (
 	github.com/grafana/grafana-aws-sdk v0.11.0
 	github.com/grafana/grafana-azure-sdk-go v1.3.1
 	github.com/grafana/grafana-plugin-sdk-go v0.142.0
-	github.com/grafana/thema v0.0.0-20220929145912-2c7c4a7bb20b
+	github.com/grafana/thema v0.0.0-20221107225215-00ad2949c7bc
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/hashicorp/go-hclog v1.0.0
 	github.com/hashicorp/go-plugin v1.4.3
@@ -110,7 +110,7 @@ require (
 	golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d
 	golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect
 	golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0
-	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
+	golang.org/x/sync v0.1.0
 	golang.org/x/time v0.0.0-20220609170525-579cf78fd858
 	golang.org/x/tools v0.1.12
 	gonum.org/v1/gonum v0.11.0
@@ -250,11 +250,13 @@ require (
 	github.com/bufbuild/connect-go v1.0.0
 	github.com/dlmiddlecote/sqlstats v1.0.2
 	github.com/drone/drone-cli v1.6.1
-	github.com/getkin/kin-openapi v0.94.0
+	github.com/getkin/kin-openapi v0.103.0
 	github.com/golang-migrate/migrate/v4 v4.7.0
 	github.com/google/go-github/v45 v45.2.0
+	github.com/grafana/codejen v0.0.2
 	github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f
 	github.com/jmoiron/sqlx v1.3.5
+	github.com/kr/pretty v0.3.0
 	github.com/matryer/is v1.4.0
 	github.com/parca-dev/parca v0.12.1
 	github.com/urfave/cli v1.22.9
@@ -285,11 +287,13 @@ require (
 	github.com/gosimple/unidecode v1.0.1 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/memberlist v0.4.0 // indirect
+	github.com/invopop/yaml v0.1.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-ieproxy v0.0.3 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/rogpeppe/go-internal v1.8.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/segmentio/asm v1.1.4 // indirect
 	go.starlark.net v0.0.0-20221020143700-22309ac47eac // indirect

go.sum

@@ -847,8 +847,9 @@ github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo
 github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM=
 github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
-github.com/getkin/kin-openapi v0.94.0 h1:bAxg2vxgnHHHoeefVdmGbR+oxtJlcv5HsJJa3qmAHuo=
 github.com/getkin/kin-openapi v0.94.0/go.mod h1:LWZfzOd7PRy8GJ1dJ6mCU6tNdSfOwRac1BUPam4aw6Q=
+github.com/getkin/kin-openapi v0.103.0 h1:F5wAtaQvPWxKCAYZ69LgHAThgu16p4u41VQtbn1U8LA=
+github.com/getkin/kin-openapi v0.103.0/go.mod h1:w4lRPHiyOdwGbOkLIyk+P0qCwlu7TXPCHD/64nSXzgE=
 github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
 github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo=
 github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0=
@@ -1347,6 +1348,8 @@ github.com/gosimple/slug v1.12.0 h1:xzuhj7G7cGtd34NXnW/yF0l+AGNfWqwgh/IXgFy7dnc=
 github.com/gosimple/slug v1.12.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ=
 github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o=
 github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc=
+github.com/grafana/codejen v0.0.2 h1:Ssp27X7SOnYxaPUTByW/6201tNV5Q60l1BSF+s3lRP8=
+github.com/grafana/codejen v0.0.2/go.mod h1:zmwwM/DRyQB7pfuBjTWII3CWtxcXh8LTwAYGfDfpR6s=
 github.com/grafana/cuetsy v0.1.1 h1:+1jaDDYCpvKlcOWJgBRbkc5+VZIClCEn5mbI+4PLZqM=
 github.com/grafana/cuetsy v0.1.1/go.mod h1:4KWkUOslwvRTpEv7wdQG0jDFTuJmU+0L9x0h4kWxa2A=
 github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f h1:FvvSVEbnGeM2bUivGmsiXTi8URJyBU7TcFEEoRe5wWI=
@@ -1366,8 +1369,8 @@ github.com/grafana/prometheus-alertmanager v0.24.1-0.20221012142027-823cd9150293
 github.com/grafana/prometheus-alertmanager v0.24.1-0.20221012142027-823cd9150293/go.mod h1:HVHqK+BVPa/tmL8EMhLCCrPt2a1GdJpEyxr5hgur2UI=
 github.com/grafana/saml v0.4.9-0.20220727151557-61cd9c9353fc h1:1PY8n+rXuBNr3r1JQhoytWDCpc+pq+BibxV0SZv+Cr4=
 github.com/grafana/saml v0.4.9-0.20220727151557-61cd9c9353fc/go.mod h1:9Zh6dWPtB3MSzTRt8fIFH60Z351QQ+s7hCU3J/tTlA4=
-github.com/grafana/thema v0.0.0-20220929145912-2c7c4a7bb20b h1:OEGzlaj04LE6Eq7aGMOh0bCplGW5rXNeSSSwgamPBEY=
-github.com/grafana/thema v0.0.0-20220929145912-2c7c4a7bb20b/go.mod h1:i3/NX50sNrwsPSAQAj56ckjQTb4biaYG/6y+zyKgpb0=
+github.com/grafana/thema v0.0.0-20221107225215-00ad2949c7bc h1:Icv777/PBaqhLmbSBSDaajDl424cbmh5ee77Du2rUFE=
+github.com/grafana/thema v0.0.0-20221107225215-00ad2949c7bc/go.mod h1:wnIJykzNiNVANl6g/Z4nkXxoMqaaH1LoG0IPNW++BEk=
 github.com/grafana/xorm v0.8.3-0.20220614223926-2fcda7565af6 h1:I9dh1MXGX0wGyxdV/Sl7+ugnki4Dfsy8lv2s5Yf887o=
 github.com/grafana/xorm v0.8.3-0.20220614223926-2fcda7565af6/go.mod h1:ZkJLEYLoVyg7amJK/5r779bHyzs2AU8f8VMiP6BM7uY=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
@@ -1552,6 +1555,8 @@ github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bS
 github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0=
 github.com/influxdata/tdigest v0.0.2-0.20210216194612-fc98d27c9e8b/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y=
 github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po=
+github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc=
+github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
 github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
 github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
@@ -2888,8 +2893,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -3376,6 +3382,7 @@ gopkg.in/yaml.v3 v3.0.0-20200603094226-e3079894b1e8/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=

kinds/gen.go

@@ -0,0 +1,157 @@
+//go:build ignore
+// +build ignore
+
+//go:generate go run gen.go
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"cuelang.org/go/cue/errors"
+	"github.com/grafana/codejen"
+	"github.com/grafana/grafana/pkg/codegen"
+	"github.com/grafana/grafana/pkg/cuectx"
+	"github.com/grafana/grafana/pkg/kindsys"
+)
+
+const sep = string(filepath.Separator)
+
+func main() {
+	if len(os.Args) > 1 {
+		fmt.Fprintf(os.Stderr, "plugin thema code generator does not currently accept any arguments\n, got %q", os.Args)
+		os.Exit(1)
+	}
+
+	// Core kinds composite code generator. Produces all generated code in
+	// grafana/grafana that derives from raw and structured core kinds.
+	coreKindsGen := codejen.JennyListWithNamer[*codegen.DeclForGen](func(decl *codegen.DeclForGen) string {
+		return decl.Meta.Common().MachineName
+	})
+
+	// All the jennies that comprise the core kinds generator pipeline
+	coreKindsGen.Append(
+		codegen.GoTypesJenny(kindsys.GoCoreKindParentPath, nil),
+		codegen.CoreStructuredKindJenny(kindsys.GoCoreKindParentPath, nil),
+		codegen.RawKindJenny(kindsys.GoCoreKindParentPath, nil),
+		codegen.BaseCoreRegistryJenny(filepath.Join("pkg", "registry", "corekind"), kindsys.GoCoreKindParentPath),
+		codegen.TSTypesJenny(kindsys.TSCoreKindParentPath, &codegen.TSTypesGeneratorConfig{
+			GenDirName: func(decl *codegen.DeclForGen) string {
+				// FIXME this hardcodes always generating to experimental dir. OK for now, but need generator fanout
+				return filepath.Join(decl.Meta.Common().MachineName, "x")
+			},
+		}),
+		codegen.TSVeneerIndexJenny(filepath.Join("packages", "grafana-schema", "src")),
+	)
+
+	coreKindsGen.AddPostprocessors(codegen.SlashHeaderMapper("kinds/gen.go"))
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "could not get working directory: %s", err)
+		os.Exit(1)
+	}
+	grootp := strings.Split(cwd, sep)
+	groot := filepath.Join(sep, filepath.Join(grootp[:len(grootp)-1]...))
+
+	rt := cuectx.GrafanaThemaRuntime()
+	var all []*codegen.DeclForGen
+
+	// structured kinddirs first
+	f := os.DirFS(filepath.Join(groot, kindsys.CoreStructuredDeclParentPath))
+	kinddirs := elsedie(fs.ReadDir(f, "."))("error reading structured fs root directory")
+	for _, ent := range kinddirs {
+		if !ent.IsDir() {
+			continue
+		}
+		rel := filepath.Join(kindsys.CoreStructuredDeclParentPath, ent.Name())
+		decl, err := kindsys.LoadCoreKind[kindsys.CoreStructuredMeta](rel, rt.Context(), nil)
+		if err != nil {
+			die(fmt.Errorf("%s is not a valid kind: %s", rel, errors.Details(err, nil)))
+		}
+		if decl.Meta.MachineName != ent.Name() {
+			die(fmt.Errorf("%s: kind's machine name (%s) must equal parent dir name (%s)", rel, decl.Meta.Name, ent.Name()))
+		}
+
+		all = append(all, elsedie(codegen.ForGen(rt, decl.Some()))(rel))
+	}
+
+	// now raw kinddirs
+	f = os.DirFS(filepath.Join(groot, kindsys.RawDeclParentPath))
+	kinddirs = elsedie(fs.ReadDir(f, "."))("error reading raw fs root directory")
+	for _, ent := range kinddirs {
+		if !ent.IsDir() {
+			continue
+		}
+		rel := filepath.Join(kindsys.RawDeclParentPath, ent.Name())
+		decl, err := kindsys.LoadCoreKind[kindsys.RawMeta](rel, rt.Context(), nil)
+		if err != nil {
+			die(fmt.Errorf("%s is not a valid kind: %s", rel, errors.Details(err, nil)))
+		}
+		if decl.Meta.MachineName != ent.Name() {
+			die(fmt.Errorf("%s: kind's machine name (%s) must equal parent dir name (%s)", rel, decl.Meta.Name, ent.Name()))
+		}
+		dfg, _ := codegen.ForGen(nil, decl.Some())
+		all = append(all, dfg)
+	}
+
+	sort.Slice(all, func(i, j int) bool {
+		return nameFor(all[i].Meta) < nameFor(all[j].Meta)
+	})
+
+	jfs, err := coreKindsGen.GenerateFS(all)
+	if err != nil {
+		die(fmt.Errorf("core kinddirs codegen failed: %w", err))
+	}
+	// for _, f := range jfs.AsFiles() {
+	// 	fmt.Println(filepath.Join(groot, f.RelativePath))
+	// }
+
+	if _, set := os.LookupEnv("CODEGEN_VERIFY"); set {
+		if err = jfs.Verify(context.Background(), groot); err != nil {
+			die(fmt.Errorf("generated code is out of sync with inputs:\n%s\nrun `make gen-cue` to regenerate", err))
+		}
+	} else if err = jfs.Write(context.Background(), groot); err != nil {
+		die(fmt.Errorf("error while writing generated code to disk:\n%s", err))
+	}
+}
+
+func nameFor(m kindsys.SomeKindMeta) string {
+	switch x := m.(type) {
+	case kindsys.RawMeta:
+		return x.Name
+	case kindsys.CoreStructuredMeta:
+		return x.Name
+	case kindsys.CustomStructuredMeta:
+		return x.Name
+	case kindsys.ComposableMeta:
+		return x.Name
+	default:
+		// unreachable so long as all the possibilities in KindMetas have switch branches
+		panic("unreachable")
+	}
+}
+
+func elsedie[T any](t T, err error) func(msg string) T {
+	if err != nil {
+		return func(msg string) T {
+			fmt.Fprintf(os.Stderr, "%s: %s\n", msg, err)
+			os.Exit(1)
+			return t
+		}
+	}
+	return func(msg string) T {
+		return t
+	}
+}
+
+func die(err error) {
+	fmt.Fprint(os.Stderr, err, "\n")
+	os.Exit(1)
+}

kinds/raw/constraint.cue

@@ -0,0 +1,7 @@
+package kind
+
+import "github.com/grafana/grafana/pkg/kindsys"
+
+// In each child directory, the set of .cue files with 'package kind'
+// must be an instance of kindsys.#Raw - a declaration of a raw kind.
+kindsys.#Raw

kinds/raw/svg/svg_kind.cue

@@ -0,0 +1,4 @@
+package kind
+
+name: "SVG"
+extensions: ["svg"]

kinds/structured/constraint.cue

@@ -0,0 +1,8 @@
+package kind
+
+import "github.com/grafana/grafana/pkg/kindsys"
+
+// In each child directory, the set of .cue files with 'package kind'
+// must be an instance of kindsys.#CoreStructured - a declaration of a
+// structured kind.
+kindsys.#CoreStructured