init.go

package crd

import (
	"context"
	"fmt"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/kubernot/wrangler/pkg/apply"
	"github.com/kubernot/wrangler/pkg/data/convert"
	"github.com/kubernot/wrangler/pkg/kv"
	"github.com/kubernot/wrangler/pkg/name"
	"github.com/kubernot/wrangler/pkg/schemas/openapi"
	"github.com/sirupsen/logrus"
	apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/rest"

	// Ensure the gvks are loaded so that apply works correctly
	_ "github.com/kubernot/wrangler/pkg/generated/controllers/apiextensions.k8s.io/v1"
)

var (
	// Ref: https://github.com/kubernetes/apiextensions-apiserver/blob/v0.28.0/pkg/apis/apiextensions/validation/validation.go#L53
	customResourceColumnDefinitionFormats = sets.NewString("int32", "int64", "float", "double", "byte", "date", "date-time", "password")
)

const CRDKind = "CustomResourceDefinition"

type Factory struct {
	wg        sync.WaitGroup
	err       error
	CRDClient clientset.Interface
	apply     apply.Apply
}

// CRD defines information about a CRD that can be used to create a CustomResourceDefinition runtime.Object
// Deprecated: Rancher does not plan to continue support for dynamically defined CRDs
type CRD struct {
	GVK          schema.GroupVersionKind
	PluralName   string
	SingularName string
	NonNamespace bool
	Schema       *apiextv1.JSONSchemaProps
	SchemaObject interface{}
	Columns      []apiextv1.CustomResourceColumnDefinition
	Status       bool
	Scale        bool
	Categories   []string
	ShortNames   []string
	Labels       map[string]string
	Annotations  map[string]string

	Override runtime.Object
}

func (c CRD) WithSchema(schema *apiextv1.JSONSchemaProps) CRD {
	c.Schema = schema
	return c
}

func (c CRD) WithSchemaFromStruct(obj interface{}) CRD {
	c.SchemaObject = obj
	return c
}

func (c CRD) WithColumn(name, path string) CRD {
	c.Columns = append(c.Columns, apiextv1.CustomResourceColumnDefinition{
		Name:     name,
		Type:     "string",
		Priority: 0,
		JSONPath: path,
	})
	return c
}

func getType(obj interface{}) reflect.Type {
	if t, ok := obj.(reflect.Type); ok {
		return t
	}

	t := reflect.TypeOf(obj)
	if t.Kind() == reflect.Ptr {
		t = t.Elem()
	}
	return t
}

func (c CRD) WithColumnsFromStruct(obj interface{}) CRD {
	c.Columns = append(c.Columns, readCustomColumns(getType(obj), ".")...)
	return c
}

func fieldName(f reflect.StructField) string {
	jsonTag := f.Tag.Get("json")
	if jsonTag == "-" {
		return ""
	}
	name := strings.Split(jsonTag, ",")[0]
	if name == "" {
		return f.Name
	}
	return name
}

func tagToColumn(f reflect.StructField, kind reflect.Kind, format, path string) (apiextv1.CustomResourceColumnDefinition, bool) {
	// Column definitions support only a subset of the full OpenAPI schema formats
	if !customResourceColumnDefinitionFormats.Has(format) {
		format = ""
	}

	c := apiextv1.CustomResourceColumnDefinition{
		Name:     f.Name,
		Type:     kindToType(kind),
		Format:   format,
		JSONPath: path,
	}

	columnDef, ok := f.Tag.Lookup("column")
	if !ok {
		return c, false
	}

	for k, v := range kv.SplitMap(columnDef, ",") {
		switch k {
		case "name":
			c.Name = v
		case "type":
			c.Type = v
		case "format":
			c.Format = v
		case "description":
			c.Description = v
		case "priority":
			p, _ := strconv.Atoi(v)
			c.Priority = int32(p)
		case "jsonpath":
			c.JSONPath = v
		}
	}

	return c, true
}

func kindToType(k reflect.Kind) string {
	// Ref: https://github.com/kubernetes/apiserver/blob/v0.28.0/pkg/endpoints/installer.go#L1178
	switch s := k.String(); s {
	case "bool", "*bool":
		return "boolean"
	case "uint8", "*uint8", "int", "*int", "int32", "*int32", "int64", "*int64", "uint32", "*uint32", "uint64", "*uint64":
		return "integer"
	case "float64", "*float64", "float32", "*float32":
		return "number"
	case "byte", "*byte":
		return "string"
	case "[]string", "[]*string":
		return "string"
	case "[]int32", "[]*int32":
		return "integer"
	default:
		return s
	}
}

type openAPISchemaProvider interface {
	OpenAPISchemaType() []string
	OpenAPISchemaFormat() string
}

func openAPISchema(t reflect.Type) (reflect.Kind, string) {
	var format string
	if o, ok := reflect.New(t).Interface().(openAPISchemaProvider); ok {
		format = o.OpenAPISchemaFormat()
		if st := o.OpenAPISchemaType(); len(st) > 0 {
			switch st[0] {
			case "string":
				return reflect.String, format
			}
		}
	}
	return reflect.Invalid, format
}

func readCustomColumns(t reflect.Type, path string) (result []apiextv1.CustomResourceColumnDefinition) {
	for i := 0; i < t.NumField(); i++ {
		f := t.Field(i)
		fieldName := fieldName(f)
		if fieldName == "" {
			continue
		}

		t := f.Type
		if t.Kind() == reflect.Ptr {
			t = t.Elem()
		}
		if kind, format := openAPISchema(t); kind != reflect.Invalid {
			if col, ok := tagToColumn(f, kind, format, path+fieldName); ok {
				result = append(result, col)
			}
		} else if t.Kind() == reflect.Struct {
			if f.Anonymous {
				result = append(result, readCustomColumns(t, path)...)
			} else {
				result = append(result, readCustomColumns(t, path+fieldName+".")...)
			}
		} else {
			if col, ok := tagToColumn(f, t.Kind(), "", path+fieldName); ok {
				result = append(result, col)
			}
		}
	}

	return result
}

func (c CRD) WithCustomColumn(columns ...apiextv1.CustomResourceColumnDefinition) CRD {
	c.Columns = append(c.Columns, columns...)
	return c
}

func (c CRD) WithStatus() CRD {
	c.Status = true
	return c
}

func (c CRD) WithScale() CRD {
	c.Scale = true
	return c
}

func (c CRD) WithCategories(categories ...string) CRD {
	c.Categories = categories
	return c
}

func (c CRD) WithGroup(group string) CRD {
	c.GVK.Group = group
	return c
}

func (c CRD) WithShortNames(shortNames ...string) CRD {
	c.ShortNames = shortNames
	return c
}

func (c CRD) ToCustomResourceDefinition() (runtime.Object, error) {
	if c.Override != nil {
		return c.Override, nil
	}

	if c.SchemaObject != nil && c.GVK.Kind == "" {
		t := getType(c.SchemaObject)
		c.GVK.Kind = t.Name()
	}

	if c.SchemaObject != nil && c.GVK.Version == "" {
		t := getType(c.SchemaObject)
		c.GVK.Version = filepath.Base(t.PkgPath())
	}

	if c.SchemaObject != nil && c.GVK.Group == "" {
		t := getType(c.SchemaObject)
		c.GVK.Group = filepath.Base(filepath.Dir(t.PkgPath()))
	}

	plural := c.PluralName
	if plural == "" {
		plural = strings.ToLower(name.GuessPluralName(c.GVK.Kind))
	}

	singular := c.SingularName
	if singular == "" {
		singular = strings.ToLower(c.GVK.Kind)
	}

	name := c.Name()

	crd := apiextv1.CustomResourceDefinition{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
		},
		Spec: apiextv1.CustomResourceDefinitionSpec{
			Group: c.GVK.Group,
			Versions: []apiextv1.CustomResourceDefinitionVersion{
				{
					Name:                     c.GVK.Version,
					Storage:                  true,
					Served:                   true,
					AdditionalPrinterColumns: c.Columns,
				},
			},
			Names: apiextv1.CustomResourceDefinitionNames{
				Plural:     plural,
				Singular:   singular,
				Kind:       c.GVK.Kind,
				Categories: c.Categories,
				ShortNames: c.ShortNames,
			},
			PreserveUnknownFields: false,
		},
	}

	if c.Schema != nil {
		crd.Spec.Versions[0].Schema = &apiextv1.CustomResourceValidation{
			OpenAPIV3Schema: c.Schema,
		}
	}

	if c.SchemaObject != nil {
		schema, err := openapi.ToOpenAPIFromStruct(c.SchemaObject)
		if err != nil {
			return nil, err
		}
		crd.Spec.Versions[0].Schema = &apiextv1.CustomResourceValidation{
			OpenAPIV3Schema: schema,
		}
	}

	// add a dummy schema because v1 requires OpenAPIV3Schema to be set
	if crd.Spec.Versions[0].Schema == nil {
		crd.Spec.Versions[0].Schema = &apiextv1.CustomResourceValidation{
			OpenAPIV3Schema: &apiextv1.JSONSchemaProps{
				Type: "object",
				Properties: map[string]apiextv1.JSONSchemaProps{
					"spec": {
						XPreserveUnknownFields: &[]bool{true}[0],
					},
					"status": {
						XPreserveUnknownFields: &[]bool{true}[0],
					},
				},
			},
		}
	}

	if c.Status {
		crd.Spec.Versions[0].Subresources = &apiextv1.CustomResourceSubresources{
			Status: &apiextv1.CustomResourceSubresourceStatus{},
		}
		if c.Scale {
			sel := "Spec.Selector"
			crd.Spec.Versions[0].Subresources.Scale = &apiextv1.CustomResourceSubresourceScale{
				SpecReplicasPath:   "Spec.Replicas",
				StatusReplicasPath: "Status.Replicas",
				LabelSelectorPath:  &sel,
			}
		}
	}

	if c.NonNamespace {
		crd.Spec.Scope = apiextv1.ClusterScoped
	} else {
		crd.Spec.Scope = apiextv1.NamespaceScoped
	}

	crd.Labels = c.Labels
	crd.Annotations = c.Annotations

	// Convert to unstructured to ensure that PreserveUnknownFields=false is set because the struct will omit false
	mapData, err := convert.EncodeToMap(crd)
	if err != nil {
		return nil, err
	}
	mapData["kind"] = CRDKind
	mapData["apiVersion"] = apiextv1.SchemeGroupVersion.String()

	return &unstructured.Unstructured{
		Object: mapData,
	}, unstructured.SetNestedField(mapData, false, "spec", "preserveUnknownFields")
}

func (c CRD) ToCustomResourceDefinitionV1Beta1() (*apiextv1beta1.CustomResourceDefinition, error) {
	toConvertCRD, err := c.ToCustomResourceDefinition()
	if err != nil {
		return nil, err
	}
	if toConvertCRD == nil {
		return nil, fmt.Errorf("cannot convert empty CRD runtime object to apiextensions v1beta1 CRD object")
	}

	unstructuredCRD, ok := toConvertCRD.(*unstructured.Unstructured)
	if !ok {
		return nil, fmt.Errorf("could not convert CRD runtime object to *unstructured.Unstructured")
	}
	var v1CRD *apiextv1.CustomResourceDefinition
	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredCRD.UnstructuredContent(), &v1CRD); err != nil {
		return nil, err
	}

	internalCRD := &apiext.CustomResourceDefinition{}
	if err := apiextv1.Convert_v1_CustomResourceDefinition_To_apiextensions_CustomResourceDefinition(v1CRD, internalCRD, nil); err != nil {
		return nil, err
	}
	v1beta1CRD := &apiextv1beta1.CustomResourceDefinition{}
	if err := apiextv1beta1.Convert_apiextensions_CustomResourceDefinition_To_v1beta1_CustomResourceDefinition(internalCRD, v1beta1CRD, nil); err != nil {
		return nil, err
	}

	// GVK is dropped during conversion, so we must add it.
	v1beta1CRD.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{
		Group:   apiextv1beta1.SchemeGroupVersion.Group,
		Version: apiextv1beta1.SchemeGroupVersion.Version,
		Kind:    CRDKind,
	})
	return v1beta1CRD, nil
}

// Name resolves the Name for the given CRD.
func (c *CRD) Name() string {
	if meta, ok := c.Override.(metav1.Object); ok && c.Override != nil {
		return meta.GetName()
	}
	kind := c.GVK.Kind
	if c.SchemaObject != nil && kind == "" {
		t := getType(c.SchemaObject)
		kind = t.Name()
	}

	group := c.GVK.Group
	if c.SchemaObject != nil && group == "" {
		t := getType(c.SchemaObject)
		group = filepath.Base(filepath.Dir(t.PkgPath()))
	}

	plural := c.PluralName
	if plural == "" {
		plural = strings.ToLower(name.GuessPluralName(kind))
	}

	return strings.ToLower(plural + "." + group)
}

func NamespacedType(name string) CRD {
	kindGroup, version := kv.Split(name, "/")
	kind, group := kv.Split(kindGroup, ".")
	kind = convert.Capitalize(kind)
	group = strings.ToLower(group)

	return FromGV(schema.GroupVersion{
		Group:   group,
		Version: version,
	}, kind)
}

func New(group, version string) CRD {
	return CRD{
		GVK: schema.GroupVersionKind{
			Group:   group,
			Version: version,
		},
		PluralName:   "",
		NonNamespace: false,
		Schema:       nil,
		SchemaObject: nil,
		Columns:      nil,
		Status:       false,
		Scale:        false,
		Categories:   nil,
		ShortNames:   nil,
	}
}

func NamespacedTypes(names ...string) (ret []CRD) {
	for _, name := range names {
		ret = append(ret, NamespacedType(name))
	}
	return
}

func NonNamespacedType(name string) CRD {
	crd := NamespacedType(name)
	crd.NonNamespace = true
	return crd
}

func NonNamespacedTypes(names ...string) (ret []CRD) {
	for _, name := range names {
		ret = append(ret, NonNamespacedType(name))
	}
	return
}

func FromGV(gv schema.GroupVersion, kind string) CRD {
	return CRD{
		GVK: gv.WithKind(kind),
	}
}

func NewFactoryFromClient(config *rest.Config) (*Factory, error) {
	apply, err := apply.NewForConfig(config)
	if err != nil {
		return nil, err
	}

	f, err := clientset.NewForConfig(config)
	if err != nil {
		return nil, err
	}

	return &Factory{
		CRDClient: f,
		apply:     apply.WithDynamicLookup().WithNoDelete(),
	}, nil
}

func (f *Factory) BatchWait() error {
	f.wg.Wait()
	return f.err
}

func (f *Factory) BatchCreateCRDs(ctx context.Context, crds ...CRD) *Factory {
	f.wg.Add(1)
	go func() {
		defer f.wg.Done()
		if _, err := f.CreateCRDs(ctx, crds...); err != nil && f.err == nil {
			f.err = err
		}
	}()
	return f
}

func (f *Factory) CreateCRDs(ctx context.Context, crds ...CRD) (map[schema.GroupVersionKind]*apiextv1.CustomResourceDefinition, error) {
	if len(crds) == 0 {
		return nil, nil
	}

	if ok, err := f.ensureAccess(ctx); err != nil {
		return nil, err
	} else if !ok {
		logrus.Infof("No access to list CRDs, assuming CRDs are pre-created.")
		return nil, err
	}

	crdStatus := map[schema.GroupVersionKind]*apiextv1.CustomResourceDefinition{}

	ready, err := f.getReadyCRDs(ctx)
	if err != nil {
		return nil, err
	}

	for _, crdDef := range crds {
		crd, err := f.createCRD(ctx, crdDef, ready)
		if err != nil {
			return nil, err
		}
		crdStatus[crdDef.GVK] = crd
	}

	ready, err = f.getReadyCRDs(ctx)
	if err != nil {
		return nil, err
	}

	for gvk, crd := range crdStatus {
		if readyCrd, ok := ready[crd.Name]; ok {
			crdStatus[gvk] = readyCrd
		} else {
			if err := f.waitCRD(ctx, crd.Name, gvk, crdStatus); err != nil {
				return nil, err
			}
		}
	}

	return crdStatus, nil
}

func (f *Factory) waitCRD(ctx context.Context, crdName string, gvk schema.GroupVersionKind, crdStatus map[schema.GroupVersionKind]*apiextv1.CustomResourceDefinition) error {
	logrus.Infof("Waiting for CRD %s to become available", crdName)
	defer logrus.Infof("Done waiting for CRD %s to become available", crdName)

	first := true
	return wait.Poll(500*time.Millisecond, 60*time.Second, func() (bool, error) {
		if !first {
			logrus.Infof("Waiting for CRD %s to become available", crdName)
		}
		first = false

		crd, err := f.CRDClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crdName, metav1.GetOptions{})
		if err != nil {
			return false, err
		}

		for _, cond := range crd.Status.Conditions {
			switch cond.Type {
			case apiextv1.Established:
				if cond.Status == apiextv1.ConditionTrue {
					crdStatus[gvk] = crd
					return true, err
				}
			case apiextv1.NamesAccepted:
				if cond.Status == apiextv1.ConditionFalse {
					logrus.Infof("Name conflict on %s: %v\n", crdName, cond.Reason)
				}
			}
		}

		return false, ctx.Err()
	})
}

func (f *Factory) createCRD(ctx context.Context, crdDef CRD, ready map[string]*apiextv1.CustomResourceDefinition) (*apiextv1.CustomResourceDefinition, error) {
	crd, err := crdDef.ToCustomResourceDefinition()
	if err != nil {
		return nil, err
	}

	meta, err := meta.Accessor(crd)
	if err != nil {
		return nil, err
	}

	logrus.Infof("Applying CRD %s", meta.GetName())
	if err := f.apply.WithOwner(crd).ApplyObjects(crd); err != nil {
		return nil, err
	}

	return f.CRDClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, meta.GetName(), metav1.GetOptions{})
}

func (f *Factory) ensureAccess(ctx context.Context) (bool, error) {
	_, err := f.CRDClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{})
	if apierrors.IsForbidden(err) {
		return false, nil
	}
	return true, err
}

func (f *Factory) getReadyCRDs(ctx context.Context) (map[string]*apiextv1.CustomResourceDefinition, error) {
	list, err := f.CRDClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{})
	if err != nil {
		return nil, err
	}

	result := map[string]*apiextv1.CustomResourceDefinition{}

	for i, crd := range list.Items {
		for _, cond := range crd.Status.Conditions {
			switch cond.Type {
			case apiextv1.Established:
				if cond.Status == apiextv1.ConditionTrue {
					result[crd.Name] = &list.Items[i]
				}
			}
		}
	}

	return result, nil
}