Skip to content

Commit

Permalink
Verify that the api-server serves the correct CRDs that this client e…
Browse files Browse the repository at this point in the history 
…xpects (#1664)

* Verify that the api-server serves the correct CRDs that this client expects

Signed-off-by: Andreas Neumann <aneumann@mesosphere.com>
  • Loading branch information
@ANeumann82
ANeumann82 committed Sep 14, 2020
1 parent 8dccafc commit 3c51361
Show file tree
Hide file tree
Showing 3 changed files with 222 additions and 35 deletions.
85 changes: 71 additions & 14 deletions pkg/kudoctl/kudoinit/crd/crds.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
"fmt"
"os"
"reflect"

apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1"
Expand Down Expand Up @@ -66,15 +67,31 @@ func (c Initializer) PreUpgradeVerify(client *kube.Client, result *verifier.Resu
return nil
}

// VerifyInstallation ensures that the CRDs are installed and have the correct and expected version
// VerifyInstallation ensures that all CRDs are installed and are the same as this CLI would install
func (c Initializer) VerifyInstallation(client *kube.Client, result *verifier.Result) error {
if err := c.verifyInstallation(client.ExtClient.ApiextensionsV1beta1(), c.Operator, result); err != nil {
apiClient := client.ExtClient.ApiextensionsV1beta1()
if err := c.verifyInstallation(apiClient, c.Operator, result); err != nil {
return err
}
if err := c.verifyInstallation(client.ExtClient.ApiextensionsV1beta1(), c.OperatorVersion, result); err != nil {
if err := c.verifyInstallation(apiClient, c.OperatorVersion, result); err != nil {
return err
}
if err := c.verifyInstallation(client.ExtClient.ApiextensionsV1beta1(), c.Instance, result); err != nil {
if err := c.verifyInstallation(apiClient, c.Instance, result); err != nil {
return err
}
return nil
}

// VerifyServedVersion ensures that the api server provides the correct version of all CRDs that this client understands
func (c Initializer) VerifyServedVersion(client *kube.Client, expectedVersion string, result *verifier.Result) error {
apiClient := client.ExtClient.ApiextensionsV1beta1()
if err := c.verifyServedVersion(apiClient, c.Operator.Name, expectedVersion, result); err != nil {
return err
}
if err := c.verifyServedVersion(apiClient, c.OperatorVersion.Name, expectedVersion, result); err != nil {
return err
}
if err := c.verifyServedVersion(apiClient, c.Instance.Name, expectedVersion, result); err != nil {
return err
}
return nil
Expand All @@ -94,6 +111,7 @@ func (c Initializer) Install(client *kube.Client) error {
return nil
}

// verifyIsNotInstalled is used to ensure that the cluster has no old KUDO version installed
func (c Initializer) verifyIsNotInstalled(client v1beta1.CustomResourceDefinitionsGetter, crd *apiextv1beta1.CustomResourceDefinition, result *verifier.Result) error {
_, err := client.CustomResourceDefinitions().Get(context.TODO(), crd.Name, v1.GetOptions{})
if err != nil {
Expand All @@ -106,21 +124,29 @@ func (c Initializer) verifyIsNotInstalled(client v1beta1.CustomResourceDefinitio
return nil
}

func (c Initializer) verifyInstallation(client v1beta1.CustomResourceDefinitionsGetter, crd *apiextv1beta1.CustomResourceDefinition, result *verifier.Result) error {
existingCrd, err := client.CustomResourceDefinitions().Get(context.TODO(), crd.Name, v1.GetOptions{})
func (c Initializer) getCrdForVerify(client v1beta1.CustomResourceDefinitionsGetter, crdName string, result *verifier.Result) (*apiextv1beta1.CustomResourceDefinition, error) {
existingCrd, err := client.CustomResourceDefinitions().Get(context.TODO(), crdName, v1.GetOptions{})
if err != nil {
if os.IsTimeout(err) {
return err
return nil, err
}
if kerrors.IsNotFound(err) {
result.AddErrors(fmt.Sprintf("CRD %s is not installed", crd.Name))
return nil
result.AddErrors(fmt.Sprintf("CRD %s is not installed", crdName))
return nil, nil
}
return fmt.Errorf("failed to retrieve CRD %s: %v", crd.Name, err)
return nil, fmt.Errorf("failed to retrieve CRD %s: %v", crdName, err)
}
if existingCrd.Spec.Version != crd.Spec.Version {
result.AddErrors(fmt.Sprintf("Installed CRD %s has invalid version %s, expected %s", crd.Name, existingCrd.Spec.Version, crd.Spec.Version))
return nil
return existingCrd, nil
}

// VerifyInstallation ensures that a single CRD is installed and is the same as this CLI would install
func (c Initializer) verifyInstallation(client v1beta1.CustomResourceDefinitionsGetter, crd *apiextv1beta1.CustomResourceDefinition, result *verifier.Result) error {
existingCrd, err := c.getCrdForVerify(client, crd.Name, result)
if err != nil || existingCrd == nil {
return err
}
if !reflect.DeepEqual(existingCrd.Spec.Versions, crd.Spec.Versions) {
result.AddErrors(fmt.Sprintf("Installed CRD versions do not match expected CRD versions (%v vs %v).", existingCrd.Spec.Versions, crd.Spec.Versions))
}
if healthy, msg, err := status.IsHealthy(existingCrd); !healthy || err != nil {
if err != nil {
Expand All @@ -129,7 +155,38 @@ func (c Initializer) verifyInstallation(client v1beta1.CustomResourceDefinitions
result.AddErrors(fmt.Sprintf("Installed CRD %s is not healthy: %v", crd.Name, msg))
return nil
}
clog.V(2).Printf("CRD %s is installed with version %s", crd.Name, existingCrd.Spec.Versions[0].Name)
clog.V(2).Printf("CRD %s is installed with versions %v", crd.Name, existingCrd.Spec.Versions)
return nil
}

// VerifyServedVersion ensures that the api server provides the correct version of a specific CRDs that this client understands
func (c Initializer) verifyServedVersion(client v1beta1.CustomResourceDefinitionsGetter, crdName, version string, result *verifier.Result) error {
existingCrd, err := c.getCrdForVerify(client, crdName, result)
if err != nil || existingCrd == nil {
return err
}
if err := health.IsHealthy(existingCrd); err != nil {
result.AddErrors(err.Error())
return nil
}

var expectedVersion *apiextv1beta1.CustomResourceDefinitionVersion
var allNames = []string{}
for _, v := range existingCrd.Spec.Versions {
v := v
allNames = append(allNames, v.Name)
if v.Name == version {
expectedVersion = &v
break
}
}
if expectedVersion == nil {
result.AddErrors(fmt.Sprintf("Expected API version %s was not found for %s, api-server only supports %v. Please update your KUDO CLI.", version, crdName, allNames))
return nil
}
if !expectedVersion.Served {
result.AddErrors(fmt.Sprintf("Expected API version %s for %s is known to api-server, but is not served. Please update your KUDO CLI.", version, crdName))
}
return nil
}

Expand Down
45 changes: 24 additions & 21 deletions pkg/kudoctl/util/kudo/kudo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,40 +58,29 @@ func NewClient(kubeConfigPath string, requestTimeout int64, validateInstall bool

// NewClient creates new KUDO Client
func NewClientForConfig(config *rest.Config, validateInstall bool) (*Client, error) {

kubeClient, err := kube.GetKubeClientForConfig(config)
if err != nil {
return nil, clog.Errorf("could not get Kubernetes client: %s", err)
}

result := verifier.NewResult()
err = crd.NewInitializer().VerifyInstallation(kubeClient, &result)
if err != nil {
return nil, fmt.Errorf("failed to run crd verification: %v", err)
}
if !result.IsValid() {
clog.V(0).Printf("KUDO CRDs are not set up correctly. Do you need to run kudo init?")

if validateInstall {
return nil, fmt.Errorf("CRDs invalid: %v", result.ErrorsAsString())
}
}

// create the kudo clientset
kudoClientset, err := versioned.NewForConfig(config)
if err != nil {
return nil, err
}

// create the kubernetes clientset
kubeClientset, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, err
}
return &Client{
client := &Client{
kudoClientset: kudoClientset,
KubeClientset: kubeClientset,
}, nil
KubeClientset: kubeClient.KubeClient,
}

validationErr := client.VerifyServedCRDs(kubeClient)
if validateInstall && validationErr != nil {
return nil, validationErr
}

return client, nil
}

// NewClientFromK8s creates KUDO client from kubernetes client interface
Expand All @@ -102,6 +91,20 @@ func NewClientFromK8s(kudo versioned.Interface, kube kubernetes.Interface) *Clie
return &result
}

func (c *Client) VerifyServedCRDs(kubeClient *kube.Client) error {
result := verifier.NewResult()
err := crd.NewInitializer().VerifyServedVersion(kubeClient, v1beta1.SchemeGroupVersion.Version, &result)
if err != nil {
return fmt.Errorf("failed to run crd verification: %v", err)
}
if !result.IsValid() {
clog.V(0).Printf("KUDO CRDs are not served in the expected version.")
return fmt.Errorf("CRDs invalid: %v", result.ErrorsAsString())
}

return nil
}

// OperatorExistsInCluster checks if a given Operator object is installed on the current k8s cluster
func (c *Client) OperatorExistsInCluster(name, namespace string) bool {
operator, err := c.kudoClientset.KudoV1beta1().Operators(namespace).Get(context.TODO(), name, v1.GetOptions{})
Expand Down
127 changes: 127 additions & 0 deletions pkg/kudoctl/util/kudo/kudo_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,17 @@ import (

"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apiextensionfake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
dynamicfake "k8s.io/client-go/dynamic/fake"
kubefake "k8s.io/client-go/kubernetes/fake"

kudoapi "github.com/kudobuilder/kudo/pkg/apis/kudo/v1beta1"
"github.com/kudobuilder/kudo/pkg/client/clientset/versioned/fake"
"github.com/kudobuilder/kudo/pkg/kudoctl/kube"
"github.com/kudobuilder/kudo/pkg/util/convert"
"github.com/kudobuilder/kudo/pkg/util/kudo"
)
Expand All @@ -25,6 +29,129 @@ func newTestSimpleK2o() *Client {
return NewClientFromK8s(fake.NewSimpleClientset(), kubefake.NewSimpleClientset())
}

func newFakeKubeClient() *kube.Client {
client := kubefake.NewSimpleClientset()
extClient := apiextensionfake.NewSimpleClientset()
dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme())

return &kube.Client{
KubeClient: client,
ExtClient: extClient,
DynamicClient: dynamicClient}
}

func TestKudoClient_ValidateServedCrds(t *testing.T) {
crdWrongVersion := apiextv1beta1.CustomResourceDefinition{
Spec: apiextv1beta1.CustomResourceDefinitionSpec{
Versions: []apiextv1beta1.CustomResourceDefinitionVersion{
{
Name: "v1beta2",
Served: true,
},
},
},
Status: apiextv1beta1.CustomResourceDefinitionStatus{
Conditions: []apiextv1beta1.CustomResourceDefinitionCondition{
{
Type: apiextv1beta1.Established,
Status: apiextv1beta1.ConditionTrue,
},
},
},
}
crdNotServed := apiextv1beta1.CustomResourceDefinition{
Spec: apiextv1beta1.CustomResourceDefinitionSpec{
Versions: []apiextv1beta1.CustomResourceDefinitionVersion{
{
Name: "v1beta1",
Served: false,
},
{
Name: "v1beta2",
Served: true,
},
},
},
Status: apiextv1beta1.CustomResourceDefinitionStatus{
Conditions: []apiextv1beta1.CustomResourceDefinitionCondition{
{
Type: apiextv1beta1.Established,
Status: apiextv1beta1.ConditionTrue,
},
},
},
}
crdUnHealthy := apiextv1beta1.CustomResourceDefinition{
Spec: apiextv1beta1.CustomResourceDefinitionSpec{
Versions: []apiextv1beta1.CustomResourceDefinitionVersion{
{
Name: "v1beta2",
Served: true,
},
},
},
Status: apiextv1beta1.CustomResourceDefinitionStatus{
Conditions: []apiextv1beta1.CustomResourceDefinitionCondition{
{
Type: apiextv1beta1.Established,
Status: apiextv1beta1.ConditionFalse,
},
{
Type: apiextv1beta1.Terminating,
Status: apiextv1beta1.ConditionTrue,
},
},
},
}

tests := []struct {
name string
crdBase *apiextv1beta1.CustomResourceDefinition
err string
}{
{name: "no crd", err: `CRDs invalid: CRD operators.kudo.dev is not installed
CRD operatorversions.kudo.dev is not installed
CRD instances.kudo.dev is not installed
`},
{name: "wrong version", crdBase: &crdWrongVersion, err: `CRDs invalid: Expected API version v1beta1 was not found for operators.kudo.dev, api-server only supports [v1beta2]. Please update your KUDO CLI.
Expected API version v1beta1 was not found for operatorversions.kudo.dev, api-server only supports [v1beta2]. Please update your KUDO CLI.
Expected API version v1beta1 was not found for instances.kudo.dev, api-server only supports [v1beta2]. Please update your KUDO CLI.
`},
{name: "not served", crdBase: &crdNotServed, err: `CRDs invalid: Expected API version v1beta1 for operators.kudo.dev is known to api-server, but is not served. Please update your KUDO CLI.
Expected API version v1beta1 for operatorversions.kudo.dev is known to api-server, but is not served. Please update your KUDO CLI.
Expected API version v1beta1 for instances.kudo.dev is known to api-server, but is not served. Please update your KUDO CLI.
`},
{name: "unhealthy", crdBase: &crdUnHealthy, err: `CRDs invalid: CRD operators.kudo.dev is not healthy ( Conditions: [{Established False 0001-01-01 00:00:00 +0000 UTC } {Terminating True 0001-01-01 00:00:00 +0000 UTC }] )
CRD operatorversions.kudo.dev is not healthy ( Conditions: [{Established False 0001-01-01 00:00:00 +0000 UTC } {Terminating True 0001-01-01 00:00:00 +0000 UTC }] )
CRD instances.kudo.dev is not healthy ( Conditions: [{Established False 0001-01-01 00:00:00 +0000 UTC } {Terminating True 0001-01-01 00:00:00 +0000 UTC }] )
`},
}

crdNames := []string{"instances.kudo.dev", "operators.kudo.dev", "operatorversions.kudo.dev"}

for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
kudoClient := newTestSimpleK2o()
kubeClient := newFakeKubeClient()

if tt.crdBase != nil {
for _, crdName := range crdNames {
crd := tt.crdBase.DeepCopy()
crd.Name = crdName
_, _ = kubeClient.ExtClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{})
}
}

err := kudoClient.VerifyServedCRDs(kubeClient)

assert.EqualError(t, err, tt.err)

})
}

}

func TestKudoClient_OperatorExistsInCluster(t *testing.T) {

obj := kudoapi.Operator{
Expand Down

0 comments on commit 3c51361

Please sign in to comment.