framework.go

// Copyright 2018 The vault-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package framework

import (
	"flag"
	"fmt"

	"github.com/coreos/vault-operator/pkg/client"
	"github.com/coreos/vault-operator/pkg/generated/clientset/versioned"
	"github.com/coreos/vault-operator/pkg/util/k8sutil"
	"github.com/coreos/vault-operator/pkg/util/probe"
	"github.com/coreos/vault-operator/test/e2e/e2eutil"

	eopk8sutil "github.com/coreos/etcd-operator/pkg/util/k8sutil"
	"github.com/sirupsen/logrus"
	appsv1beta1 "k8s.io/api/apps/v1beta1"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/client-go/kubernetes"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

var (
	// The global framework variable used by all upgrade tests
	Global           *Framework
	etcdOperatorName = "etcd-operator"
)

// Framework struct contains the various clients and other information needed to run the upgrade tests
type Framework struct {
	KubeClient     kubernetes.Interface
	Config         *restclient.Config
	VaultsCRClient versioned.Interface
	Namespace      string
	oldVOPImage    string
	newVOPImage    string
	eopImage       string
}

// Setup initializes the Global framework by initializing necessary clients and sets up the etcd-operator
func Setup() error {
	kubeconfig := flag.String("kubeconfig", "", "kube config path, e.g. $HOME/.kube/config")
	oldVOPImage := flag.String("old-vop-image", "", "operator image, e.g. quay.io/coreos/vault-operator-dev:latest")
	newVOPImage := flag.String("new-vop-image", "", "operator image, e.g. quay.io/coreos/vault-operator-dev:master")
	eopImage := flag.String("etcd-operator-image", "quay.io/coreos/etcd-operator:v0.8.3", "etcd operator image, e.g. quay.io/coreos/etcd-operator")
	ns := flag.String("namespace", "", "e2e test namespace")
	flag.Parse()

	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
	if err != nil {
		return fmt.Errorf("failed to build config from kubeconfig: %v", err)
	}
	kubeClient, err := kubernetes.NewForConfig(config)
	if err != nil {
		return fmt.Errorf("faild to create kube client: %v", err)
	}
	vaultsCRClient := client.MustNew(config)

	Global = &Framework{
		KubeClient:     kubeClient,
		Config:         config,
		VaultsCRClient: vaultsCRClient,
		Namespace:      *ns,
		oldVOPImage:    *oldVOPImage,
		newVOPImage:    *newVOPImage,
		eopImage:       *eopImage,
	}

	if err := Global.deployEtcdOperatorPod(); err != nil {
		return fmt.Errorf("failed to setup etcd operator: %v", err)
	}
	return nil
}

// TearDown removes the etcd-operator pod
func TearDown() error {
	err := Global.KubeClient.CoreV1().Pods(Global.Namespace).Delete(etcdOperatorName, k8sutil.CascadeDeleteBackground())
	if err != nil {
		return fmt.Errorf("failed to delete pod: %v", err)
	}
	Global = nil
	logrus.Info("e2e teardown successfully")
	return nil
}

// CreateOperatorDeployment creates a vault operator deployment with the specified name
func (f *Framework) CreateOperatorDeployment(name string) error {
	d := &appsv1beta1.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: f.Namespace,
		},
		Spec: appsv1beta1.DeploymentSpec{
			Strategy: appsv1beta1.DeploymentStrategy{
				Type: appsv1beta1.RollingUpdateDeploymentStrategyType,
				RollingUpdate: &appsv1beta1.RollingUpdateDeployment{
					MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 1},
					MaxSurge:       &intstr.IntOrString{Type: intstr.Int, IntVal: 1},
				},
			},
			Selector: &metav1.LabelSelector{MatchLabels: e2eutil.PodLabelForOperator(name)},
			Template: v1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: e2eutil.PodLabelForOperator(name),
				},
				Spec: v1.PodSpec{
					Containers: []v1.Container{{
						Name:            name,
						Image:           f.oldVOPImage,
						ImagePullPolicy: v1.PullAlways,
						Env: []v1.EnvVar{
							{
								Name:      "MY_POD_NAMESPACE",
								ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.namespace"}},
							},
							{
								Name:      "MY_POD_NAME",
								ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.name"}},
							},
						},
						ReadinessProbe: &v1.Probe{
							Handler: v1.Handler{
								HTTPGet: &v1.HTTPGetAction{
									Path: probe.HTTPReadyzEndpoint,
									Port: intstr.IntOrString{Type: intstr.Int, IntVal: 8080},
								},
							},
							InitialDelaySeconds: 3,
							PeriodSeconds:       3,
							FailureThreshold:    3,
						},
					}},
				},
			},
		},
	}
	_, err := f.KubeClient.AppsV1beta1().Deployments(f.Namespace).Create(d)
	if err != nil {
		return fmt.Errorf("failed to create deployment: %v", err)
	}
	return nil
}

// DeleteOperatorDeployment deletes the vault operator deployment with the specified name, waits for all its pods to be removed and deletes the Endpoint resource used for the leader election lock
func (f *Framework) DeleteOperatorDeployment(name string) error {
	err := f.KubeClient.AppsV1beta1().Deployments(f.Namespace).Delete(name, k8sutil.CascadeDeleteBackground())
	if err != nil {
		return fmt.Errorf("failed to delete deployment: %v", err)
	}

	lo := metav1.ListOptions{
		LabelSelector: labels.SelectorFromSet(e2eutil.PodLabelForOperator(name)).String(),
	}
	_, err = e2eutil.WaitPodsDeletedCompletely(f.KubeClient, f.Namespace, 3, lo)
	if err != nil {
		return fmt.Errorf("failed to wait for operator pods to be completely removed: %v", err)
	}
	// The deleted operator will not actively release the Endpoints lock causing a non-leader candidate to timeout for the lease duration: 15s
	// Deleting the Endpoints resource simulates the leader actively releasing the lock so that the next candidate avoids the timeout.
	// TODO: change this if we change to use another kind of lock, e.g. configmap.
	return f.KubeClient.CoreV1().Endpoints(f.Namespace).Delete("vault-operator", metav1.NewDeleteOptions(0))
}

func (f *Framework) UpgradeOperator(name string) error {
	uf := func(d *appsv1beta1.Deployment) {
		d.Spec.Template.Spec.Containers[0].Image = f.newVOPImage
	}
	// TODO: Put PatchDeployment into vault's k8sutil
	err := eopk8sutil.PatchDeployment(f.KubeClient, f.Namespace, name, uf)
	if err != nil {
		return fmt.Errorf("failed to patch deployment: %v", err)
	}

	lo := metav1.ListOptions{
		LabelSelector: labels.SelectorFromSet(e2eutil.PodLabelForOperator(name)).String(),
	}
	pods, err := e2eutil.WaitPodsWithImageDeleted(f.KubeClient, f.Namespace, f.oldVOPImage, 3, lo)
	if err != nil {
		return fmt.Errorf("failed to wait for pods (%v) with old image (%v) to get deleted: %v", pods, f.oldVOPImage, err)
	}
	return e2eutil.WaitUntilOperatorReady(f.KubeClient, f.Namespace, name)
}

func (f *Framework) deployEtcdOperatorPod() error {
	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:   etcdOperatorName,
			Labels: e2eutil.PodLabelForOperator(etcdOperatorName),
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:            etcdOperatorName,
					Image:           f.eopImage,
					ImagePullPolicy: v1.PullAlways,
					Command:         []string{"etcd-operator", "--create-crd=false"},
					Env: []v1.EnvVar{
						{
							Name:      "MY_POD_NAMESPACE",
							ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.namespace"}},
						},
						{
							Name:      "MY_POD_NAME",
							ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.name"}},
						},
					},
					ReadinessProbe: &v1.Probe{
						Handler: v1.Handler{
							HTTPGet: &v1.HTTPGetAction{
								Path: probe.HTTPReadyzEndpoint,
								Port: intstr.IntOrString{Type: intstr.Int, IntVal: 8080},
							},
						},
						InitialDelaySeconds: 3,
						PeriodSeconds:       3,
						FailureThreshold:    3,
					},
				},
			},
			RestartPolicy: v1.RestartPolicyNever,
		},
	}

	_, err := f.KubeClient.CoreV1().Pods(f.Namespace).Create(pod)
	if err != nil {
		return fmt.Errorf("failed to create pod: %v", err)
	}
	return e2eutil.WaitUntilOperatorReady(f.KubeClient, f.Namespace, etcdOperatorName)
}