thema: Flatten schemas to one-dimensional array

.golangci.yml

@@ -19,12 +19,10 @@ linters:
     - dogsled
     - errcheck
     - exportloopref
-    - funlen
     - gochecknoinits
     - gocritic
     - goconst
     - gocyclo
-    - gofumpt
     - goimports
     - revive
     - gomnd

assignable.go

@@ -29,7 +29,7 @@ func AssignableTo(sch Schema, T any) error {
 	rt := sch.Lineage().Runtime()
 	rt.rl()
 	defer rt.ru()
-	return assignable(sch.Underlying(), T)
+	return assignable(sch.Underlying().LookupPath(pathSchDef), T)
 }
 
 // ErrPointerDepth indicates that a Go type having pointer indirection depth > 1

bind.go

@@ -0,0 +1,151 @@
+package thema
+
+import (
+	"fmt"
+
+	"cuelang.org/go/cue"
+	cerrors "cuelang.org/go/cue/errors"
+	"cuelang.org/go/cue/token"
+	"github.com/cockroachdb/errors"
+	terrors "github.com/grafana/thema/errors"
+	"github.com/grafana/thema/internal/compat"
+)
+
+// maybeLineage is an intermediate processing structure used to validate
+// inputs as actual lineages
+//
+// it's important that these flags are populated in order to avoid false negatives.
+// no system ensures this, it's all human reasoning
+type maybeLineage struct {
+	// user lineage definition, NOT unified with thema.#Lineage
+	raw cue.Value
+
+	// user lineage definition, unified with thema.#Lineage
+	uni cue.Value
+
+	// original input cue.Value representing the lineage. May or may not be unified
+	// with thema.#Lineage
+	orig cue.Value
+
+	rt *Runtime
+
+	// pos of the original input for the lineage
+	pos token.Pos
+
+	// bind options passed by the caller
+	cfg *bindConfig
+
+	schlist []*schemaDef
+
+	allv []SyntacticVersion
+
+	// The raw input value is the root of a package instance
+	// rawIsPackage bool
+}
+
+func (ml *maybeLineage) checkGoValidity(cfg *bindConfig) error {
+	schiter, err := ml.uni.LookupPath(cue.MakePath(cue.Hid("_sortedSchemas", "github.com/grafana/thema"))).List()
+	if err != nil {
+		panic(fmt.Sprintf("unreachable - should have already verified schemas field exists and is list: %+v", cerrors.Details(err, nil)))
+	}
+	schpath := cue.MakePath(cue.Hid("_#schema", "github.com/grafana/thema"))
+	vpath := cue.MakePath(cue.Str("version"))
+
+	var previous *schemaDef
+	for schiter.Next() {
+		// Only thing not natively enforced in CUE is that the #SchemaDef.version field is concrete
+		svval := schiter.Value().LookupPath(vpath)
+		iter, err := svval.List()
+		if err != nil {
+			panic(fmt.Sprintf("unreachable - should have already verified #SchemaDef.version field exists and is list: %+v", err))
+		}
+		for iter.Next() {
+			if !iter.Value().IsConcrete() {
+				return errors.Mark(mkerror(iter.Value(), "#SchemaDef.version must have concrete major and minor versions"), terrors.ErrInvalidLineage)
+			}
+		}
+		sch := &schemaDef{}
+		err = svval.Decode(&sch.v)
+		if err != nil {
+			panic(fmt.Sprintf("unreachable - could not decode syntactic version: %+v", err))
+		}
+
+		sch.ref = schiter.Value()
+		sch.def = sch.ref.LookupPath(schpath)
+		if previous != nil && !cfg.skipbuggychecks {
+			compaterr := compat.ThemaCompatible(sch.def, previous.def)
+			if (sch.v[1] == 0 && compaterr == nil) || (sch.v[1] != 0 && compaterr != nil) {
+				return &compatInvariantError{
+					rawlin:    ml.uni,
+					violation: [2]SyntacticVersion{previous.v, sch.v},
+					detail:    compaterr,
+				}
+			}
+		}
+
+		ml.schlist = append(ml.schlist, sch)
+		ml.allv = append(ml.allv, sch.v)
+		previous = sch
+	}
+
+	return nil
+}
+
+func (ml *maybeLineage) checkExists(cfg *bindConfig) error {
+	p := ml.raw.Path().String()
+	// The candidate lineage must exist.
+	// TODO can we do any better with contextualizing these errors?
+	if !ml.raw.Exists() {
+		if p != "" {
+			return errors.Mark(errors.Newf("not a lineage: no cue value at path %q", p), terrors.ErrValueNotExist)
+		}
+
+		return errors.WithStack(terrors.ErrValueNotExist)
+	}
+	return nil
+}
+
+func (ml *maybeLineage) checkLineageShape(cfg *bindConfig) error {
+	// Check certain paths specifically, because these are common getting started errors of just arranging
+	// CUE statements in the right way that deserve more targeted guidance
+	for _, path := range []string{"name", "schemas"} {
+		val := ml.raw.LookupPath(cue.MakePath(cue.Str(path)))
+		if !val.Exists() {
+			return errors.Mark(mkerror(ml.raw, "not a lineage, missing #Lineage.%s", path), terrors.ErrValueNotALineage)
+		}
+		if !val.IsConcrete() {
+			return errors.Mark(mkerror(val, "invalid lineage, #Lineage.%s must be concrete", path), terrors.ErrInvalidLineage)
+		}
+	}
+
+	// The candidate lineage must be an instance of #Lineage. However, we can't validate the whole
+	// structure, because lenses will fail validation. This is because we currently expect them to be written:
+	//
+	// {
+	// 		input: _
+	// 		result: {
+	// 			foo: input.foo
+	// 		}
+	// }
+	//
+	// means that those structures won't pass Validate until we've injected an actual object there.
+	if err := ml.uni.Validate(cue.Final()); err != nil {
+		return errors.Mark(cerrors.Promote(err, "not an instance of thema.#Lineage"), terrors.ErrInvalidLineage)
+	}
+
+	return nil
+}
+
+// Checks the validity properties of lineages that are expressible natively in CUE.
+func (ml *maybeLineage) checkNativeValidity(cfg *bindConfig) error {
+	// The candidate lineage must be error-free.
+	// TODO replace this with Err, this check isn't actually what we want up here. Only schemas themselves must be cycle-free
+	if err := ml.raw.Validate(cue.Concrete(false)); err != nil {
+		return errors.Mark(cerrors.Promote(err, "lineage is invalid"), terrors.ErrInvalidLineage)
+	}
+	if err := ml.uni.Validate(cue.Concrete(false)); err != nil {
+		return errors.Mark(cerrors.Promote(err, "lineage is invalid"), terrors.ErrInvalidLineage)
+	}
+
+	return nil
+}

cmd/thema/lineage.go

@@ -30,6 +30,9 @@ func setupLineageCommand(cmd *cobra.Command) {
 
 	gc := new(genCommand)
 	gc.setup(linCmd)
+
+	fc := new(fixCommand)
+	fc.setup(linCmd)
 }
 
 func toSubpath(subpath string, f *ast.File) (*ast.File, error) {

cmd/thema/lineage_bump.go

@@ -2,12 +2,7 @@ package main
 
 import (
 	"fmt"
-	"os"
 
-	"cuelang.org/go/cue/ast"
-	"github.com/grafana/thema"
-	"github.com/grafana/thema/encoding/cue"
-	tastutil "github.com/grafana/thema/internal/astutil"
 	"github.com/spf13/cobra"
 )
 
@@ -36,44 +31,49 @@ func (bc *bumpCommand) setup(cmd *cobra.Command) {
 	lineageBumpCmd.Flags().BoolVar(&bc.maj, "major", false, "Bump the major version (breaking change) instead of the minor version")
 	lineageBumpCmd.Flags().BoolVar(&bc.maj, "no-fill", false, "Do not pre-fill the new schema with the prior schema")
 	lineageBumpCmd.PreRunE = bc.lla.validateLineageInput
-	lineageBumpCmd.Run = bc.run
-}
-
-func (bc *bumpCommand) run(cmd *cobra.Command, args []string) {
-	if err := bc.do(cmd, args); err != nil {
-		fmt.Fprintf(cmd.ErrOrStderr(), "%s\n", err)
-		os.Exit(1)
+	// lineageBumpCmd.Run = bc.run
+	// TODO un-hide this once cue encoding package is refactored
+	lineageBumpCmd.Hidden = true
+	lineageBumpCmd.Run = func(cmd *cobra.Command, args []string) {
+		fmt.Fprint(cmd.OutOrStdout(), "bump subcommand is disabled, pending refactor")
 	}
 }
 
-func (bc *bumpCommand) do(cmd *cobra.Command, args []string) error {
-	lv := thema.LatestVersion(bc.lla.dl.lin)
-	lsch := thema.SchemaP(bc.lla.dl.lin, lv)
-	// TODO UGH EVAL
-	schlit := tastutil.Format(lsch.Underlying().Eval())
-
-	var err error
-	var nlin ast.Node
-	if bc.maj {
-		nlin = bc.lla.dl.lin.Underlying().Source()
-		err = cue.InsertSchemaNodeAs(nlin, tastutil.ToExpr(schlit), thema.SV(lv[0]+1, 0))
-		if err != nil {
-			return err
-		}
-	} else {
-		nlin, err = cue.Append(bc.lla.dl.lin, lsch.Underlying())
-		if err != nil {
-			return err
-		}
-	}
-
-	b, err := tastutil.FmtNode(tastutil.ToExpr(nlin))
-	if err != nil {
-		return err
-	}
-
-	// TODO write back to subpath
+// func (bc *bumpCommand) run(cmd *cobra.Command, args []string) {
+// 	if err := bc.do(cmd, args); err != nil {
+// 		fmt.Fprintf(cmd.ErrOrStderr(), "%s\n", err)
+// 		os.Exit(1)
+// 	}
+// }
 
-	fmt.Fprint(cmd.OutOrStdout(), string(b))
-	return nil
-}
+// func (bc *bumpCommand) do(cmd *cobra.Command, args []string) error {
+// 	lv := thema.LatestVersion(bc.lla.dl.lin)
+// 	lsch := thema.SchemaP(bc.lla.dl.lin, lv)
+// 	// TODO UGH EVAL
+// 	schlit := tastutil.Format(lsch.Underlying().Eval())
+//
+// 	var err error
+// 	var nlin ast.Node
+// 	if bc.maj {
+// 		nlin = bc.lla.dl.lin.Underlying().Source()
+// 		err = cue.InsertSchemaNodeAs(nlin, tastutil.ToExpr(schlit), thema.SV(lv[0]+1, 0))
+// 		if err != nil {
+// 			return err
+// 		}
+// 	} else {
+// 		nlin, err = cue.Append(bc.lla.dl.lin, lsch.Underlying())
+// 		if err != nil {
+// 			return err
+// 		}
+// 	}
+//
+// 	b, err := tastutil.FmtNode(tastutil.ToExpr(nlin))
+// 	if err != nil {
+// 		return err
+// 	}
+//
+// 	// TODO write back to subpath
+//
+// 	fmt.Fprint(cmd.OutOrStdout(), string(b))
+// 	return nil
+// }

cmd/thema/lineage_fix.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"cuelang.org/go/cue"
+	cuenc "github.com/grafana/thema/encoding/cue"
+	tastutil "github.com/grafana/thema/internal/astutil"
+	"github.com/spf13/cobra"
+)
+
+var lineageFixCmd = &cobra.Command{
+	Use:   "fix",
+	Args:  cobra.MaximumNArgs(0),
+	Short: "Rewrite legacy lineage definitions to new standards",
+	Long: `Rewrite legacy lineage definitions to current standards.
+`,
+}
+
+type fixCommand struct {
+	lla *lineageLoadArgs
+}
+
+func (fc *fixCommand) setup(cmd *cobra.Command) {
+	cmd.AddCommand(lineageFixCmd)
+	fc.lla = new(lineageLoadArgs)
+
+	lineageFixCmd.PersistentFlags().StringVarP(&fc.lla.inputLinFilePath, "lineage", "l", ".", "path to .cue file or package containing legacy lineage to rewrite")
+	lineageFixCmd.MarkFlagRequired("lineage")
+	lineageFixCmd.PersistentFlags().StringVarP(&fc.lla.lincuepath, "path", "p", "", "CUE expression for path to the lineage object within file, if not root")
+	fc.lla.skipBindLineage = true
+
+	lineageFixCmd.PreRunE = fc.lla.validateLineageInput
+	lineageFixCmd.Run = fc.run
+}
+
+func (fc *fixCommand) run(cmd *cobra.Command, args []string) {
+	if err := fc.do(cmd, args); err != nil {
+		fmt.Fprintf(cmd.ErrOrStderr(), "%s\n", err)
+		os.Exit(1)
+	}
+}
+
+func (fc *fixCommand) do(cmd *cobra.Command, args []string) error {
+	f, err := cuenc.RewriteLegacyLineage(ctx.BuildInstance(fc.lla.dl.binst), cue.ParsePath(fc.lla.lincuepath))
+	if err != nil {
+		return err
+	}
+
+	b, err := tastutil.FmtNode(f)
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(f.Filename, b, 0666)
+}

cmd/thema/lineage_init.go

@@ -15,7 +15,6 @@ import (
 	"cuelang.org/go/encoding/jsonschema"
 	"cuelang.org/go/encoding/openapi"
 	"cuelang.org/go/encoding/yaml"
-	"github.com/grafana/thema"
 	"github.com/grafana/thema/encoding/cue"
 	tastutil "github.com/grafana/thema/internal/astutil"
 	"github.com/spf13/cobra"
@@ -175,12 +174,13 @@ func (ic *initCommand) runEmpty(cmd *cobra.Command, args []string) {
 		return
 	}
 
+	// TODO uncomment after re-export
 	// Have to re-insert because comments get lost somehow by NewLineage()
-	err = cue.InsertSchemaNodeAs(linf, expr, thema.SV(0, 0))
-	if err != nil {
-		ic.err = err
-		return
-	}
+	// err = cue.InsertSchemaNodeAs(linf, expr, thema.SV(0, 0))
+	// if err != nil {
+	// 	ic.err = err
+	// 	return
+	// }
 
 	linf, err = toSubpath(ic.cuepath, linf)
 	if err != nil {