Merge pull request #130 from grafana/lineage/is-append-only-subsume

Lineage: First version of IsAppendOnly

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/grafana/cuetsy v0.1.8
 	github.com/labstack/echo/v4 v4.9.1
 	github.com/matryer/moq v0.2.7
+	github.com/sergi/go-diff v1.3.1
 	github.com/spf13/cobra v1.4.0
 	github.com/stretchr/testify v1.8.1
 	github.com/xeipuuv/gojsonschema v1.2.0

go.sum

@@ -1,4 +1,6 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cuelang.org/go v0.5.0 h1:D6N0UgTGJCOxFKU8RU+qYvavKNsVc/+ZobmifStVJzU=
+cuelang.org/go v0.5.0/go.mod h1:okjJBHFQFer+a41sAe2SaGm1glWS8oEb6CmJvn5Zdws=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
@@ -235,8 +237,11 @@ github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFo
 github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
 github.com/sdboyer/cue v0.5.0-beta.2.0.20221218111347-341999f48bdb h1:X6XJsprVDQnlG4vT5TVb+cRlGMU78L/IKej8Q6SDFGY=
 github.com/sdboyer/cue v0.5.0-beta.2.0.20221218111347-341999f48bdb/go.mod h1:okjJBHFQFer+a41sAe2SaGm1glWS8oEb6CmJvn5Zdws=
+github.com/sdboyer/cue v0.5.0-beta.2.0.20230526103728-b5c4d8dd99fb h1:geH2xQLSIbLbpm5GH1MisHqfIw6JKYNs+OWewSuyWD0=
+github.com/sdboyer/cue v0.5.0-beta.2.0.20230526103728-b5c4d8dd99fb/go.mod h1:okjJBHFQFer+a41sAe2SaGm1glWS8oEb6CmJvn5Zdws=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
+github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
+github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=

instance_test.go

@@ -1,13 +1,14 @@
 package thema
 
 import (
-	"cuelang.org/go/cue"
 	"encoding/json"
 	"fmt"
 	"io"
 	"strings"
 	"testing"
 
+	"cuelang.org/go/cue"
+
 	"github.com/grafana/thema/internal/txtartest/vanilla"
 
 	"cuelang.org/go/cue/cuecontext"
@@ -28,7 +29,7 @@ func TestInstance_Translate(t *testing.T) {
 			return
 		}
 
-		lin, lerr := bindTxtarLineage(tc, rt)
+		lin, lerr := bindTxtarLineage(tc, rt, "lineagePath")
 		require.NoError(tc, lerr)
 
 		for from := lin.First(); from != nil; from = from.Successor() {

internal/cuetil/equal.go

@@ -0,0 +1,18 @@
+package cuetil
+
+import (
+	"cuelang.org/go/cue"
+)
+
+// Equal reports nil when the two cue values subsume each other or an error otherwise
+func Equal(val1 cue.Value, val2 cue.Value) error {
+	if err := val1.Subsume(val2, cue.Raw()); err != nil {
+		return err
+	}
+
+	if err := val2.Subsume(val1, cue.Raw()); err != nil {
+		return err
+	}
+
+	return nil
+}

lineage.go

@@ -189,6 +189,17 @@ func (lin *baseLineage) First() Schema {
 	return lin.allsch[0]
 }
 
+// All returns all Schemas in the lineage sorted by version (0.0 being the first
+// element). Thema requires that all valid lineages contain at least one schema,
+// so this is guaranteed to contain at least one element.
+func (lin *baseLineage) All() []Schema {
+	schemas := make([]Schema, len(lin.allsch))
+	for i, s := range lin.allsch {
+		schemas[i] = s
+	}
+	return schemas
+}
+
 // Underlying returns the cue.Value of the entire lineage.
 func (lin *baseLineage) Underlying() cue.Value {
 	isValidLineage(lin)
@@ -272,3 +283,29 @@ type unaryConvLineage[T Assignee] struct {
 func (lin *unaryConvLineage[T]) TypedSchema() TypedSchema[T] {
 	return lin.tsch
 }
+
+// IsAppendOnly returns nil if the new lineage only contains new schemas compared to the old one.
+// It returns an error if old schemas are updated or deleted.
+func IsAppendOnly(oldLineage Lineage, newLineage Lineage) error {
+	oldSchemas := oldLineage.All()
+	newSchemas := newLineage.All()
+
+	if len(newSchemas) < len(oldSchemas) {
+		return fmt.Errorf("schemas can't be deleted once published")
+	}
+
+	for i, schema := range oldSchemas {
+		schemaPath := "schema"
+		oldSchema := schema.Underlying()
+		x := oldSchema.LookupPath(cue.ParsePath(schemaPath))
+
+		newSchema := newSchemas[i].Underlying()
+		y := newSchema.LookupPath(cue.ParsePath(schemaPath))
+
+		if err := cuetil.Equal(x, y); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

lineage_test.go

@@ -24,7 +24,7 @@ func TestBindLineage(t *testing.T) {
 	rt := NewRuntime(ctx)
 
 	test.Run(t, func(tc *vanilla.Test) {
-		lin, err := bindTxtarLineage(tc, rt)
+		lin, err := bindTxtarLineage(tc, rt, "lineagePath")
 		if testing.Short() && tc.HasTag("slow") {
 			t.Skip("case is tagged #slow, skipping for -short")
 		}
@@ -67,7 +67,7 @@ func TestInvalidLineages(t *testing.T) {
 	rt := NewRuntime(ctx)
 
 	test.Run(t, func(tc *vanilla.Test) {
-		_, err := bindTxtarLineage(tc, rt)
+		_, err := bindTxtarLineage(tc, rt, "lineagePath")
 		if testing.Short() && tc.HasTag("slow") {
 			tc.Skip("case is tagged #slow, skipping for -short")
 		}
@@ -80,7 +80,77 @@ func TestInvalidLineages(t *testing.T) {
 	})
 }
 
-func bindTxtarLineage(t *vanilla.Test, rt *Runtime) (Lineage, error) {
+func TestIsAppendOnly(t *testing.T) {
+	test := vanilla.TxTarTest{
+		Root: "./testdata/isappendonly/valid",
+		Name: "isappendonly",
+		ToDo: map[string]string{
+			"isappendonly/valid/withconstraints": "Subsume doesn't support constraints using built-in validators",
+			"isappendonly/valid/disjunction":     "Subsume requires the Final() option to consider two complex disjunctions as equal but this creates false negatives",
+			"isappendonly/valid/maps":            "Subsume requires the Final() option to consider two maps as equal but this creates false negatives",
+		},
+	}
+
+	ctx := cuecontext.New()
+	rt := NewRuntime(ctx)
+
+	test.Run(t, func(tc *vanilla.Test) {
+		if testing.Short() && tc.HasTag("slow") {
+			t.Skip("case is tagged #slow, skipping for -short")
+		}
+
+		lin1, err := bindTxtarLineage(tc, rt, "firstLin")
+		if err != nil {
+			tc.Fatalf("error binding first lineage: %+v", err)
+		}
+
+		lin2, err := bindTxtarLineage(tc, rt, "secondLin")
+		if err != nil {
+			tc.Fatalf("error binding second lineage: %+v", err)
+		}
+
+		err = IsAppendOnly(lin1, lin2)
+		if err != nil {
+			tc.Fatalf("IsAppendOnly returned an error: %+v", err)
+		}
+	})
+}
+
+func TestIsAppendOnlyFail(t *testing.T) {
+	test := vanilla.TxTarTest{
+		Root: "./testdata/isappendonly/invalid",
+		Name: "isappendonly-fail",
+	}
+
+	ctx := cuecontext.New()
+	rt := NewRuntime(ctx)
+
+	test.Run(t, func(tc *vanilla.Test) {
+		if testing.Short() && tc.HasTag("slow") {
+			t.Skip("case is tagged #slow, skipping for -short")
+		}
+
+		lin1, err := bindTxtarLineage(tc, rt, "firstLin")
+		if err != nil {
+			tc.Fatalf("error binding first lineage: %+v", err)
+		}
+
+		lin2, err := bindTxtarLineage(tc, rt, "secondLin")
+		if err != nil {
+			tc.Fatalf("error binding second lineage: %+v", err)
+		}
+
+		err = IsAppendOnly(lin1, lin2)
+		if err == nil {
+			tc.Fatalf("expected error from known invalid updates")
+		}
+
+		// TODO more verbose error output, should include CUE line-level analysis
+		tc.WriteErrors(errors.Promote(err, "IsAppendOnly fail"))
+	})
+}
+
+func bindTxtarLineage(t *vanilla.Test, rt *Runtime, path string) (Lineage, error) {
 	if rt == nil {
 		rt = NewRuntime(cuecontext.New())
 	}
@@ -89,14 +159,14 @@ func bindTxtarLineage(t *vanilla.Test, rt *Runtime) (Lineage, error) {
 	t.Helper()
 	inst := t.Instance()
 	val := ctx.BuildInstance(inst)
-	if p, ok := t.Value("lineagePath"); ok {
+	if p, ok := t.Value(path); ok {
 		pp := cue.ParsePath(p)
 		if len(pp.Selectors()) == 0 {
-			t.Fatalf("%q is not a valid value for the #lineagePath key", p)
+			t.Fatalf("%q is not a valid value for the #%s key", p, path)
 		}
 		val = val.LookupPath(pp)
 		if !val.Exists() {
-			t.Fatalf("path %q specified in #lineagePath does not exist in input cue instance", p)
+			t.Fatalf("path %q specified in #%s does not exist in input cue instance", p, path)
 		}
 	}
 

surface.go

@@ -72,6 +72,10 @@ type Lineage interface {
 	// Otherwise, it is probably preferable to pick an explicit version number.
 	Latest() Schema
 
+	// All returns all Schemas in the lineage. Thema requires that all valid lineages
+	// contain at least one schema, so this is guaranteed to contain at least one element.
+	All() []Schema
+
 	// Runtime returns the thema.Runtime instance with which this lineage was built.
 	Runtime() *Runtime
 

testdata/isappendonly/invalid/boundaries.txtar

@@ -0,0 +1,27 @@
+#firstLin: lin1
+#secondLin: lin2
+-- in.cue --
+import "github.com/grafana/thema"
+
+lin1: thema.#Lineage
+lin1: name: "boundaries"
+lin1: schemas: [{
+	version: [0, 0]
+	schema: {
+		anInt: uint32 & >0 & <=24 | *12
+	}
+}]
+
+lin2: thema.#Lineage
+lin2: name: "boundaries"
+lin2: schemas: [{
+	version: [0, 0]
+	schema: {
+		anInt: uint32 & >0 & <=14 | *12
+	}
+}]
+-- out/isappendonly-fail --
+field anInt not present in {anInt:*12 | >0 & <=24 & int}:
+    ../../../lineage.cue:247:10
+    ./in.cue:7:10
+missing field "anInt"

testdata/isappendonly/invalid/default.txtar

@@ -0,0 +1,27 @@
+#firstLin: lin1
+#secondLin: lin2
+-- in.cue --
+import "github.com/grafana/thema"
+
+lin1: thema.#Lineage
+lin1: name: "defaultchange"
+lin1: schemas: [{
+	version: [0, 0]
+	schema: {
+		aunion: *"foo" | "bar" | "baz"
+	}
+}]
+
+lin2: thema.#Lineage
+lin2: name: "defaultchange"
+lin2: schemas: [{
+    version: [0, 0]
+    schema: {
+        aunion: "foo" | *"bar" | "baz"
+    }
+}]
+-- out/isappendonly-fail --
+field aunion not present in {aunion:*"bar" | "foo" | "baz"}:
+    ../../../lineage.cue:247:10
+    ./in.cue:16:13
+missing field "aunion"

testdata/isappendonly/invalid/embedref.txtar

@@ -0,0 +1,39 @@
+#firstLin: lin1
+#secondLin: lin2
+-- in.cue --
+import "github.com/grafana/thema"
+
+lin1: thema.#Lineage
+lin1: name: "embedref"
+lin1: schemas: [{
+	version: [0, 0]
+	schema: {
+		#EmbedRef
+
+        #EmbedRef: {
+            refField1: string
+            refField2: 42
+        }
+	}
+}]
+
+lin2: thema.#Lineage
+lin2: name: "embedref"
+lin2: schemas: [{
+    version: [0, 0]
+    schema: {
+        #EmbedRef
+
+        #EmbedRef: {
+            refField1: string
+            refField2: 1
+        }
+    }
+}]
+-- out/isappendonly-fail --
+field #EmbedRef not present in {#EmbedRef:{refField1:string,refField2:1},refField1:string,refField2:1}:
+    ../../../lineage.cue:247:10
+    ./in.cue:21:13
+field refField2 not present in {refField1:string,refField2:1}:
+    ./in.cue:24:20
+missing field "#EmbedRef"

testdata/isappendonly/invalid/nested.txtar

@@ -0,0 +1,33 @@
+#firstLin: lin1
+#secondLin: lin2
+-- in.cue --
+import "github.com/grafana/thema"
+
+lin1: thema.#Lineage
+lin1: name: "nested"
+lin1: schemas: [{
+	version: [0, 0]
+	schema: {
+		anObject: {
+            aField: string
+        }
+	}
+}]
+
+lin2: thema.#Lineage
+lin2: name: "nested"
+lin2: schemas: [{
+    version: [0, 0]
+    schema: {
+		anObject: {
+            aField: string
+            aNewOptionalField?: string
+        }
+    }
+}]
+-- out/isappendonly-fail --
+field anObject not present in {anObject:{aField:string}}:
+    ../../../lineage.cue:247:10
+    ./in.cue:18:13
+field not allowed in closed struct: aNewOptionalField
+missing field "anObject"