Use go-bindata to embed YAML files with CRDs. (#1211)

* Remove the unused crdVersion from CRD.

It was never used (verified by eyeballing `git log -p` all the way
down), we don't seem to know what it's for, and it has no counterpart in
the structs.

* Rename the CRD files.

This is in preparation for moving to `controller-gen`, which is somewhat
opinionated in that it does not allow us to pick the exact file names.

I guess we do not really care what these are called anyway - the only
place where these names show up is this integration test.

* goimports

* Use go-bindata to embed YAML files with CRDs.

Also add go-bindata to the `generate` target, and a README.

* Remove option to write manifests from test.

They are the authoritative *source* now.

Keep the test around as a way of making sure the file generated by
go-bindata is up-to-date.

* Update README.

Makefile

@@ -84,6 +84,10 @@ deploy-clean:
 .PHONY: generate
 # Generate code
 generate:
+ifeq (, $(shell which go-bindata))
+	go get github.com/go-bindata/go-bindata/go-bindata
+endif
+	go-bindata -pkg crd -o pkg/kudoctl/kudoinit/crd/bindata.go -ignore README.md config/crds
 	./hack/update_codegen.sh
 
 .PHONY: generate-clean

config/crds/README.md

@@ -1,4 +1,4 @@
 # CRD manifests
 
-Note that some these files can be auto-generated from the authoritative definition in the `kubectl kudo init`
-implementation by running `go test -tags integration ./pkg/kudoctl/cmd -v -update`.
+These files are currently the authoritative definition for the CRDs (some of these
+are used as source by `kubectl kudo init`).

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/dustinkirkland/golang-petname v0.0.0-20170921220637-d3c2ba80e75e
 	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-bindata/go-bindata v3.1.2+incompatible // indirect
 	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
 	github.com/google/btree v1.0.0 // indirect

go.sum

@@ -120,6 +120,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
 github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/go-bindata/go-bindata v3.1.2+incompatible h1:5vjJMVhowQdPzjE1LdxyFF7YFTXg5IgGVW4gBr5IbvE=
+github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg=

pkg/kudoctl/cmd/init_integration_test.go

@@ -56,49 +56,19 @@ const (
 
 func TestCrds_Config(t *testing.T) {
 	crds := crd.NewInitializer()
-
-	if *updateGolden {
-		err := writeManifest(operatorFileName, crds.Operator)
-		if err != nil {
-			t.Errorf("Operator file override failed: %v", err)
-		}
-		err = writeManifest(operatorVersionFileName, crds.OperatorVersion)
-		if err != nil {
-			t.Errorf("OperatorVersion file override failed: %v", err)
-		}
-		err = writeManifest(instanceFileName, crds.Instance)
-		if err != nil {
-			t.Errorf("Instance file override failed: %v", err)
-		}
-	}
-
 	assertManifestFileMatch(t, operatorFileName, crds.Operator)
 	assertManifestFileMatch(t, operatorVersionFileName, crds.OperatorVersion)
 	assertManifestFileMatch(t, instanceFileName, crds.Instance)
 }
 
-func writeManifest(fileName string, expectedObject runtime.Object) error {
-	expectedContent, err := runtimeObjectAsBytes(expectedObject)
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("Updating file %s", fileName)
-	path := filepath.Join(manifestsDir, fileName)
-	if err := ioutil.WriteFile(path, expectedContent, 0644); err != nil {
-		return fmt.Errorf("failed to update config file: %s", err)
-	}
-	return nil
-}
-
 func assertManifestFileMatch(t *testing.T, fileName string, expectedObject runtime.Object) {
 	expectedContent, err := runtimeObjectAsBytes(expectedObject)
 	assert.Nil(t, err)
 	path := filepath.Join(manifestsDir, fileName)
 	of, err := ioutil.ReadFile(path)
 	assert.Nil(t, err)
 
-	assert.Equal(t, string(expectedContent), string(of), "manifest file does not match the existing one")
+	assert.Equal(t, string(expectedContent), string(of), fmt.Sprintf("embedded file %s does not match the source, run 'make generate'", fileName))
 }
 
 func runtimeObjectAsBytes(o runtime.Object) ([]byte, error) {

pkg/kudoctl/kudoinit/crd/README.md

@@ -0,0 +1,4 @@
+# Embedded CRDs
+
+The CRDs provided by this package are embedded into the binary through the file `bindata.go`,
+generated by `go-bindata` invoked from the `make generate` target.

pkg/kudoctl/kudoinit/crd/crds.go

@@ -4,12 +4,10 @@ package crd
 import (
 	"fmt"
 	"os"
-	"strings"
 
 	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"sigs.k8s.io/yaml"
@@ -19,11 +17,6 @@ import (
 	"github.com/kudobuilder/kudo/pkg/kudoctl/kudoinit"
 )
 
-const (
-	group      = "kudo.dev"
-	crdVersion = "v1beta1"
-)
-
 // Ensure kudoinit.InitStep is implemented
 var _ kudoinit.InitStep = &Initializer{}
 
@@ -37,9 +30,9 @@ type Initializer struct {
 // CRDs returns the runtime.Object representation of all the CRDs KUDO requires
 func NewInitializer() Initializer {
 	return Initializer{
-		Operator:        operatorCrd(),
-		OperatorVersion: operatorVersionCrd(),
-		Instance:        instanceCrd(),
+		Operator:        embeddedCRD("config/crds/kudo.dev_operators.yaml"),
+		OperatorVersion: embeddedCRD("config/crds/kudo.dev_operatorversions.yaml"),
+		Instance:        embeddedCRD("config/crds/kudo.dev_instances.yaml"),
 	}
 }
 
@@ -113,168 +106,12 @@ func (c Initializer) install(client v1beta1.CustomResourceDefinitionsGetter, crd
 	return err
 }
 
-// operatorCrd provides definition of the operator CRD
-func operatorCrd() *apiextv1beta1.CustomResourceDefinition {
-	maintainers := map[string]apiextv1beta1.JSONSchemaProps{
-		"name":  {Type: "string"},
-		"email": {Type: "string"},
-	}
-
-	crd := generateCrd("Operator", "operators")
-	specProps := map[string]apiextv1beta1.JSONSchemaProps{
-		"description":       {Type: "string"},
-		"kubernetesVersion": {Type: "string"},
-		"kudoVersion":       {Type: "string"},
-		"maintainers": {Type: "array",
-			Items: &apiextv1beta1.JSONSchemaPropsOrArray{Schema: &apiextv1beta1.JSONSchemaProps{
-				Type:       "object",
-				Properties: maintainers,
-			}, JSONSchemas: []apiextv1beta1.JSONSchemaProps{}},
-		},
-		"url": {Type: "string"},
-	}
-
-	validationProps := map[string]apiextv1beta1.JSONSchemaProps{
-		"apiVersion": {Type: "string"},
-		"kind":       {Type: "string"},
-		"metadata":   {Type: "object"},
-		"spec":       {Properties: specProps, Type: "object"},
-		"status":     {Type: "object"},
-	}
-	crd.Spec.Validation = &apiextv1beta1.CustomResourceValidation{
-		OpenAPIV3Schema: &apiextv1beta1.JSONSchemaProps{Type: "object",
-			Properties: validationProps,
-		},
-	}
-	return crd
-}
-
-// operatorVersionCrd provides definition of the operatorversion crd
-func operatorVersionCrd() *apiextv1beta1.CustomResourceDefinition {
-	crd := generateCrd("OperatorVersion", "operatorversions")
-	paramProps := map[string]apiextv1beta1.JSONSchemaProps{
-		"default":     {Type: "string", Description: "Default is a default value if no parameter is provided by the instance."},
-		"description": {Type: "string", Description: "Description captures a longer description of how the parameter will be used."},
-		"displayName": {Type: "string", Description: "DisplayName can be used by UIs."},
-		"name":        {Type: "string", Description: "Name is the string that should be used in the template file for example, if `name: COUNT` then using the variable in a spec like: \n spec:   replicas:  {{ .Params.COUNT }}"},
-		"required":    {Type: "boolean", Description: "Required specifies if the parameter is required to be provided by all instances, or whether a default can suffice."},
-		"trigger":     {Type: "string", Description: "Trigger identifies the plan that gets executed when this parameter changes in the Instance object. Default is `update` if a plan with that name exists, otherwise it's `deploy`."},
-	}
-	taskProps := map[string]apiextv1beta1.JSONSchemaProps{
-		"name": {Type: "string"},
-		"kind": {Type: "string"},
-		"spec": {Type: "object"},
-	}
-	specProps := map[string]apiextv1beta1.JSONSchemaProps{
-		"appVersion":       {Type: "string"},
-		"connectionString": {Type: "string", Description: "ConnectionString defines a templated string that can be used to connect to an instance of the Operator."},
-		"operator":         {Type: "object"},
-		"parameters": {
-			Type: "array",
-			Items: &apiextv1beta1.JSONSchemaPropsOrArray{Schema: &apiextv1beta1.JSONSchemaProps{
-				Type:       "object",
-				Properties: paramProps,
-			}, JSONSchemas: []apiextv1beta1.JSONSchemaProps{}},
-		},
-		"plans": {Type: "object", Description: "Plans maps a plan name to a plan."},
-		"tasks": {
-			Type:        "array",
-			Description: "List of all tasks available in this OperatorVersion.",
-			Items: &apiextv1beta1.JSONSchemaPropsOrArray{Schema: &apiextv1beta1.JSONSchemaProps{
-				Type:       "object",
-				Properties: taskProps,
-			}, JSONSchemas: []apiextv1beta1.JSONSchemaProps{}},
-		},
-		"templates": {Type: "object", Description: "Templates is a list of references to YAML templates located in the templates folder and later referenced from tasks."},
-		"upgradableFrom": {
-			Type:        "array",
-			Description: "UpgradableFrom lists all OperatorVersions that can upgrade to this OperatorVersion.",
-			Items:       &apiextv1beta1.JSONSchemaPropsOrArray{Schema: &apiextv1beta1.JSONSchemaProps{Type: "object"}, JSONSchemas: []apiextv1beta1.JSONSchemaProps{}},
-		},
-		"version": {Type: "string"},
-	}
-
-	validationProps := map[string]apiextv1beta1.JSONSchemaProps{
-		"apiVersion": {Type: "string"},
-		"kind":       {Type: "string"},
-		"metadata":   {Type: "object"},
-		"spec":       {Properties: specProps, Type: "object"},
-		"status":     {Type: "object"},
-	}
-
-	crd.Spec.Validation = &apiextv1beta1.CustomResourceValidation{
-		OpenAPIV3Schema: &apiextv1beta1.JSONSchemaProps{Type: "object",
-			Properties: validationProps,
-		},
-	}
-	return crd
-}
-
-// instanceCrd provides the Instance CRD manifest for printing
-func instanceCrd() *apiextv1beta1.CustomResourceDefinition {
-	crd := generateCrd("Instance", "instances")
-	specProps := map[string]apiextv1beta1.JSONSchemaProps{
-		"operatorVersion": {Type: "object", Description: "OperatorVersion specifies a reference to a specific OperatorVersion object."},
-		"parameters":      {Type: "object"},
-	}
-	statusProps := map[string]apiextv1beta1.JSONSchemaProps{
-		"planStatus":       {Type: "object"},
-		"aggregatedStatus": {Type: "object"},
-	}
-
-	validationProps := map[string]apiextv1beta1.JSONSchemaProps{
-		"apiVersion": {Type: "string"},
-		"kind":       {Type: "string"},
-		"metadata":   {Type: "object"},
-		"spec":       {Properties: specProps, Type: "object"},
-		"status": {
-			Type:       "object",
-			Properties: statusProps,
-		},
-	}
-
-	crd.Spec.Validation = &apiextv1beta1.CustomResourceValidation{
-		OpenAPIV3Schema: &apiextv1beta1.JSONSchemaProps{Type: "object",
-			Properties: validationProps,
-		},
-	}
-
-	crd.Spec.Subresources = &apiextv1beta1.CustomResourceSubresources{Status: &apiextv1beta1.CustomResourceSubresourceStatus{}}
-	return crd
-}
-
-// generateCrd provides a generic CRD object to be configured
-func generateCrd(kind string, plural string) *apiextv1beta1.CustomResourceDefinition {
-	plural = strings.ToLower(plural)
-	name := plural + "." + group
-
-	crd := &apiextv1beta1.CustomResourceDefinition{
-		ObjectMeta: v1.ObjectMeta{
-			Name: name,
-		},
-		Spec: apiextv1beta1.CustomResourceDefinitionSpec{
-			Group:   group,
-			Version: crdVersion,
-			Names: apiextv1beta1.CustomResourceDefinitionNames{
-				Plural:     plural,
-				Singular:   strings.ToLower(kind),
-				ShortNames: nil,
-				Kind:       kind,
-			},
-			Scope: "Namespaced",
-		},
-		Status: apiextv1beta1.CustomResourceDefinitionStatus{
-			Conditions:     []apiextv1beta1.CustomResourceDefinitionCondition{},
-			StoredVersions: []string{},
-		},
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "CustomResourceDefinition",
-			APIVersion: "apiextensions.k8s.io/v1beta1",
-		},
+func embeddedCRD(path string) *apiextv1beta1.CustomResourceDefinition {
+	operatorYaml := MustAsset(path)
+	crd := &apiextv1beta1.CustomResourceDefinition{}
+	err := yaml.UnmarshalStrict(operatorYaml, crd)
+	if err != nil {
+		panic(fmt.Sprintf("cannot unmarshal embedded content of %s: %v", path, err))
 	}
-	// below is needed if we support 1.15 CRD in v1beta1, it is deprecated within the 1.15
-	// for 1.16 it is removed and functions as if preserve == false
-	// preserveFields := false
-	// crd.Spec.PreserveUnknownFields = &preserveFields
 	return crd
 }

tools.go

@@ -5,6 +5,7 @@
 package tools
 
 import (
+	_ "github.com/go-bindata/go-bindata/go-bindata"
 	_ "k8s.io/code-generator/cmd/client-gen"
 	_ "k8s.io/code-generator/cmd/deepcopy-gen"
 	_ "k8s.io/code-generator/cmd/defaulter-gen"