From b7a33511e53b9821ef8f5be2754f4d7fa0564bae Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Sun, 6 Jan 2019 22:30:56 +0800 Subject: [PATCH 1/2] Clean up local volum provisioner e2e tests --- test/e2e/storage/BUILD | 1 - test/e2e/storage/persistent_volumes-local.go | 673 ------------------- 2 files changed, 674 deletions(-) diff --git a/test/e2e/storage/BUILD b/test/e2e/storage/BUILD index e151e87e7371..b18f6c52c685 100644 --- a/test/e2e/storage/BUILD +++ b/test/e2e/storage/BUILD @@ -82,7 +82,6 @@ go_library( "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/github.com/prometheus/common/model:go_default_library", "//vendor/google.golang.org/api/googleapi:go_default_library", - "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/test/e2e/storage/persistent_volumes-local.go b/test/e2e/storage/persistent_volumes-local.go index 75f68ff506a0..08d4cf040017 100644 --- a/test/e2e/storage/persistent_volumes-local.go +++ b/test/e2e/storage/persistent_volumes-local.go @@ -18,24 +18,19 @@ package storage import ( "fmt" - "path" "path/filepath" "strconv" "strings" - "sync" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "sigs.k8s.io/yaml" appsv1 "k8s.io/api/apps/v1" "k8s.io/api/core/v1" - rbacv1beta1 "k8s.io/api/rbac/v1beta1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" @@ -137,24 +132,6 @@ const ( testFileContent = "test-file-content" testSCPrefix = "local-volume-test-storageclass" - // Following are constants used for provisioner e2e tests. - // - // testServiceAccount is the service account for bootstrapper - testServiceAccount = "local-storage-admin" - // volumeConfigName is the configmap passed to bootstrapper and provisioner - volumeConfigName = "local-volume-config" - // provisioner image used for e2e tests - provisionerImageName = "quay.io/external_storage/local-volume-provisioner:v2.1.0" - // provisioner daemonSetName name - daemonSetName = "local-volume-provisioner" - // provisioner default mount point folder - provisionerDefaultMountRoot = "/mnt/local-storage" - // provisioner node/pv cluster role binding - nodeBindingName = "local-storage:provisioner-node-binding" - pvBindingName = "local-storage:provisioner-pv-binding" - systemRoleNode = "system:node" - systemRolePVProvisioner = "system:persistent-volume-provisioner" - // A sample request size testRequestSize = "10Mi" @@ -398,107 +375,6 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { }) }) - Context("Local volume provisioner [Serial]", func() { - var volumePath string - - BeforeEach(func() { - setupStorageClass(config, &immediateMode) - setupLocalVolumeProvisioner(config) - volumePath = path.Join(config.discoveryDir, fmt.Sprintf("vol-%v", string(uuid.NewUUID()))) - setupLocalVolumeProvisionerMountPoint(config, volumePath, config.node0) - }) - - AfterEach(func() { - cleanupLocalVolumeProvisionerMountPoint(config, volumePath, config.node0) - cleanupLocalVolumeProvisioner(config) - cleanupStorageClass(config) - }) - - It("should create and recreate local persistent volume", func() { - By("Starting a provisioner daemonset") - createProvisionerDaemonset(config) - - By("Waiting for a PersistentVolume to be created") - oldPV, err := waitForLocalPersistentVolume(config.client, volumePath) - Expect(err).NotTo(HaveOccurred()) - - // Create a persistent volume claim for local volume: the above volume will be bound. - By("Creating a persistent volume claim") - claim, err := config.client.CoreV1().PersistentVolumeClaims(config.ns).Create(newLocalClaim(config)) - Expect(err).NotTo(HaveOccurred()) - err = framework.WaitForPersistentVolumeClaimPhase( - v1.ClaimBound, config.client, claim.Namespace, claim.Name, framework.Poll, 1*time.Minute) - Expect(err).NotTo(HaveOccurred()) - - claim, err = config.client.CoreV1().PersistentVolumeClaims(config.ns).Get(claim.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - Expect(claim.Spec.VolumeName).To(Equal(oldPV.Name)) - - // Delete the persistent volume claim: file will be cleaned up and volume be re-created. - By("Deleting the persistent volume claim to clean up persistent volume and re-create one") - writeCmd := createWriteCmd(volumePath, testFile, testFileContent, DirectoryLocalVolumeType) - err = issueNodeCommand(config, writeCmd, config.node0) - Expect(err).NotTo(HaveOccurred()) - err = config.client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, &metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred()) - - By("Waiting for a new PersistentVolume to be re-created") - newPV, err := waitForLocalPersistentVolume(config.client, volumePath) - Expect(err).NotTo(HaveOccurred()) - Expect(newPV.UID).NotTo(Equal(oldPV.UID)) - fileDoesntExistCmd := createFileDoesntExistCmd(volumePath, testFile) - err = issueNodeCommand(config, fileDoesntExistCmd, config.node0) - Expect(err).NotTo(HaveOccurred()) - - By("Deleting provisioner daemonset") - deleteProvisionerDaemonset(config) - }) - It("should not create local persistent volume for filesystem volume that was not bind mounted", func() { - - directoryPath := filepath.Join(config.discoveryDir, "notbindmount") - By("Creating a directory, not bind mounted, in discovery directory") - mkdirCmd := fmt.Sprintf("mkdir -p %v -m 777", directoryPath) - err := issueNodeCommand(config, mkdirCmd, config.node0) - Expect(err).NotTo(HaveOccurred()) - - By("Starting a provisioner daemonset") - createProvisionerDaemonset(config) - - By("Allowing provisioner to run for 30s and discover potential local PVs") - time.Sleep(30 * time.Second) - - By("Examining provisioner logs for not an actual mountpoint message") - provisionerPodName := findProvisionerDaemonsetPodName(config) - logs, err := framework.GetPodLogs(config.client, config.ns, provisionerPodName, "" /*containerName*/) - Expect(err).NotTo(HaveOccurred(), - "Error getting logs from pod %s in namespace %s", provisionerPodName, config.ns) - - expectedLogMessage := "Path \"/mnt/local-storage/notbindmount\" is not an actual mountpoint" - Expect(strings.Contains(logs, expectedLogMessage)).To(BeTrue()) - - By("Deleting provisioner daemonset") - deleteProvisionerDaemonset(config) - }) - It("should discover dynamically created local persistent volume mountpoint in discovery directory", func() { - By("Starting a provisioner daemonset") - createProvisionerDaemonset(config) - - By("Creating a volume in discovery directory") - dynamicVolumePath := path.Join(config.discoveryDir, fmt.Sprintf("vol-%v", string(uuid.NewUUID()))) - setupLocalVolumeProvisionerMountPoint(config, dynamicVolumePath, config.node0) - - By("Waiting for the PersistentVolume to be created") - _, err := waitForLocalPersistentVolume(config.client, dynamicVolumePath) - Expect(err).NotTo(HaveOccurred()) - - By("Deleting provisioner daemonset") - deleteProvisionerDaemonset(config) - - By("Deleting volume in discovery directory") - cleanupLocalVolumeProvisionerMountPoint(config, dynamicVolumePath, config.node0) - }) - }) - Context("StatefulSet with pod affinity [Slow]", func() { var testVols map[string][]*localTestVolume const ( @@ -556,158 +432,8 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { }) }) - Context("Stress with local volume provisioner [Serial]", func() { - var testVols [][]string - - const ( - volsPerNode = 10 // Make this non-divisable by volsPerPod to increase changes of partial binding failure - volsPerPod = 3 - podsFactor = 4 - ) - - BeforeEach(func() { - setupStorageClass(config, &waitMode) - setupLocalVolumeProvisioner(config) - - testVols = [][]string{} - for i, node := range config.nodes { - By(fmt.Sprintf("Setting up local volumes on node %q", node.Name)) - paths := []string{} - for j := 0; j < volsPerNode; j++ { - volumePath := path.Join(config.discoveryDir, fmt.Sprintf("vol-%v", string(uuid.NewUUID()))) - setupLocalVolumeProvisionerMountPoint(config, volumePath, &config.nodes[i]) - paths = append(paths, volumePath) - } - testVols = append(testVols, paths) - } - - By("Starting the local volume provisioner") - createProvisionerDaemonset(config) - }) - - AfterEach(func() { - By("Deleting provisioner daemonset") - deleteProvisionerDaemonset(config) - - for i, paths := range testVols { - for _, volumePath := range paths { - cleanupLocalVolumeProvisionerMountPoint(config, volumePath, &config.nodes[i]) - } - } - cleanupLocalVolumeProvisioner(config) - cleanupStorageClass(config) - }) - - It("should use be able to process many pods and reuse local volumes", func() { - var ( - podsLock sync.Mutex - // Have one extra pod pending - numConcurrentPods = volsPerNode/volsPerPod*len(config.nodes) + 1 - totalPods = numConcurrentPods * podsFactor - numCreated = 0 - numFinished = 0 - pods = map[string]*v1.Pod{} - ) - - // Create pods gradually instead of all at once because scheduler has - // exponential backoff - // TODO: this is still a bit slow because of the provisioner polling period - By(fmt.Sprintf("Creating %v pods periodically", numConcurrentPods)) - stop := make(chan struct{}) - go wait.Until(func() { - podsLock.Lock() - defer podsLock.Unlock() - - if numCreated >= totalPods { - // Created all the pods for the test - return - } - - if len(pods) > numConcurrentPods/2 { - // Too many outstanding pods - return - } - - for i := 0; i < numConcurrentPods; i++ { - pvcs := []*v1.PersistentVolumeClaim{} - for j := 0; j < volsPerPod; j++ { - pvc := framework.MakePersistentVolumeClaim(makeLocalPVCConfig(config, DirectoryLocalVolumeType), config.ns) - pvc, err := framework.CreatePVC(config.client, config.ns, pvc) - framework.ExpectNoError(err) - pvcs = append(pvcs, pvc) - } - - pod := framework.MakeSecPod(config.ns, pvcs, false, "sleep 1", false, false, selinuxLabel, nil) - pod, err := config.client.CoreV1().Pods(config.ns).Create(pod) - Expect(err).NotTo(HaveOccurred()) - pods[pod.Name] = pod - numCreated++ - } - }, 2*time.Second, stop) - - defer func() { - close(stop) - podsLock.Lock() - defer podsLock.Unlock() - - for _, pod := range pods { - if err := deletePodAndPVCs(config, pod); err != nil { - framework.Logf("Deleting pod %v failed: %v", pod.Name, err) - } - } - }() - - By("Waiting for all pods to complete successfully") - err := wait.PollImmediate(time.Second, 5*time.Minute, func() (done bool, err error) { - podsList, err := config.client.CoreV1().Pods(config.ns).List(metav1.ListOptions{}) - if err != nil { - return false, err - } - - podsLock.Lock() - defer podsLock.Unlock() - - for _, pod := range podsList.Items { - switch pod.Status.Phase { - case v1.PodSucceeded: - // Delete pod and its PVCs - if err := deletePodAndPVCs(config, &pod); err != nil { - return false, err - } - delete(pods, pod.Name) - numFinished++ - framework.Logf("%v/%v pods finished", numFinished, totalPods) - case v1.PodFailed: - case v1.PodUnknown: - return false, fmt.Errorf("pod %v is in %v phase", pod.Name, pod.Status.Phase) - } - } - - return numFinished == totalPods, nil - }) - Expect(err).ToNot(HaveOccurred()) - }) - }) }) -func deletePodAndPVCs(config *localTestConfig, pod *v1.Pod) error { - framework.Logf("Deleting pod %v", pod.Name) - if err := config.client.CoreV1().Pods(config.ns).Delete(pod.Name, nil); err != nil { - return err - } - - // Delete PVCs - for _, vol := range pod.Spec.Volumes { - pvcSource := vol.VolumeSource.PersistentVolumeClaim - if pvcSource != nil { - if err := framework.DeletePersistentVolumeClaim(config.client, pvcSource.ClaimName, config.ns); err != nil { - return err - } - } - } - return nil -} - type makeLocalPodWith func(config *localTestConfig, volume *localTestVolume, nodeName string) *v1.Pod func testPodWithNodeConflict(config *localTestConfig, testVolType localVolumeType, nodeName string, makeLocalPodFunc makeLocalPodWith, bindingMode storagev1.VolumeBindingMode) { @@ -1226,18 +952,6 @@ func createLocalPVCsPVs(config *localTestConfig, volumes []*localTestVolume, mod } } -func makeLocalPod(config *localTestConfig, volume *localTestVolume, cmd string) *v1.Pod { - pod := framework.MakeSecPod(config.ns, []*v1.PersistentVolumeClaim{volume.pvc}, false, cmd, false, false, selinuxLabel, nil) - if pod == nil { - return pod - } - if volume.localVolumeType == BlockLocalVolumeType { - // Block e2e tests require utilities for writing to block devices (e.g. dd), and nginx has this utilities. - pod.Spec.Containers[0].Image = imageutils.GetE2EImage(imageutils.Nginx) - } - return pod -} - func makeLocalPodWithNodeAffinity(config *localTestConfig, volume *localTestVolume, nodeName string) (pod *v1.Pod) { pod = framework.MakeSecPod(config.ns, []*v1.PersistentVolumeClaim{volume.pvc}, false, "", false, false, selinuxLabel, nil) if pod == nil { @@ -1285,16 +999,6 @@ func makeLocalPodWithNodeName(config *localTestConfig, volume *localTestVolume, return } -// createSecPod should be used when Pod requires non default SELinux labels -func createSecPod(config *localTestConfig, volume *localTestVolume, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions) (*v1.Pod, error) { - pod, err := framework.CreateSecPod(config.client, config.ns, []*v1.PersistentVolumeClaim{volume.pvc}, false, "", hostIPC, hostPID, seLinuxLabel, nil, framework.PodStartShortTimeout) - podNodeName, podNodeNameErr := podNodeName(config, pod) - Expect(podNodeNameErr).NotTo(HaveOccurred()) - framework.Logf("Security Context POD %q created on Node %q", pod.Name, podNodeName) - Expect(podNodeName).To(Equal(config.node0.Name)) - return pod, err -} - func createLocalPod(config *localTestConfig, volume *localTestVolume, fsGroup *int64) (*v1.Pod, error) { By("Creating a pod") return framework.CreateSecPod(config.client, config.ns, []*v1.PersistentVolumeClaim{volume.pvc}, false, "", false, false, selinuxLabel, fsGroup, framework.PodStartShortTimeout) @@ -1368,13 +1072,6 @@ func testReadFileContent(testFileDir string, testFile string, testFileContent st Expect(readOut).To(ContainSubstring(testFileContent)) } -// Create command to verify that the file doesn't exist -// to be executed via hostexec Pod on the node with the local PV -func createFileDoesntExistCmd(testFileDir string, testFile string) string { - testFilePath := filepath.Join(testFileDir, testFile) - return fmt.Sprintf("[ ! -e %s ]", testFilePath) -} - // Execute a read or write command in a pod. // Fail on error func podRWCmdExec(pod *v1.Pod, cmd string) string { @@ -1402,332 +1099,6 @@ func setupLocalVolumesPVCsPVs( return testVols } -func setupLocalVolumeProvisioner(config *localTestConfig) { - By("Bootstrapping local volume provisioner") - createServiceAccount(config) - createProvisionerClusterRoleBinding(config) - utils.PrivilegedTestPSPClusterRoleBinding(config.client, config.ns, false /* teardown */, []string{testServiceAccount}) - createVolumeConfigMap(config) - - for _, node := range config.nodes { - By(fmt.Sprintf("Initializing local volume discovery base path on node %v", node.Name)) - mkdirCmd := fmt.Sprintf("mkdir -p %v -m 777", config.discoveryDir) - err := issueNodeCommand(config, mkdirCmd, &node) - Expect(err).NotTo(HaveOccurred()) - } -} - -func cleanupLocalVolumeProvisioner(config *localTestConfig) { - By("Cleaning up cluster role binding") - deleteClusterRoleBinding(config) - utils.PrivilegedTestPSPClusterRoleBinding(config.client, config.ns, true /* teardown */, []string{testServiceAccount}) - - for _, node := range config.nodes { - By(fmt.Sprintf("Removing the test discovery directory on node %v", node.Name)) - removeCmd := fmt.Sprintf("[ ! -e %v ] || rm -r %v", config.discoveryDir, config.discoveryDir) - err := issueNodeCommand(config, removeCmd, &node) - Expect(err).NotTo(HaveOccurred()) - } -} - -func setupLocalVolumeProvisionerMountPoint(config *localTestConfig, volumePath string, node *v1.Node) { - By(fmt.Sprintf("Creating local directory at path %q", volumePath)) - mkdirCmd := fmt.Sprintf("mkdir %v -m 777", volumePath) - err := issueNodeCommand(config, mkdirCmd, node) - Expect(err).NotTo(HaveOccurred()) - - By(fmt.Sprintf("Mounting local directory at path %q", volumePath)) - mntCmd := fmt.Sprintf("sudo mount --bind %v %v", volumePath, volumePath) - err = issueNodeCommand(config, mntCmd, node) - Expect(err).NotTo(HaveOccurred()) -} - -func cleanupLocalVolumeProvisionerMountPoint(config *localTestConfig, volumePath string, node *v1.Node) { - By(fmt.Sprintf("Unmounting the test mount point from %q", volumePath)) - umountCmd := fmt.Sprintf("[ ! -e %v ] || sudo umount %v", volumePath, volumePath) - err := issueNodeCommand(config, umountCmd, node) - Expect(err).NotTo(HaveOccurred()) - - By("Removing the test mount point") - removeCmd := fmt.Sprintf("[ ! -e %v ] || rm -r %v", volumePath, volumePath) - err = issueNodeCommand(config, removeCmd, node) - Expect(err).NotTo(HaveOccurred()) - - By("Cleaning up persistent volume") - pv, err := findLocalPersistentVolume(config.client, volumePath) - Expect(err).NotTo(HaveOccurred()) - if pv != nil { - err = config.client.CoreV1().PersistentVolumes().Delete(pv.Name, &metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred()) - } -} - -func createServiceAccount(config *localTestConfig) { - serviceAccount := v1.ServiceAccount{ - TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ServiceAccount"}, - ObjectMeta: metav1.ObjectMeta{Name: testServiceAccount, Namespace: config.ns}, - } - _, err := config.client.CoreV1().ServiceAccounts(config.ns).Create(&serviceAccount) - Expect(err).NotTo(HaveOccurred()) -} - -// createProvisionerClusterRoleBinding creates two cluster role bindings for local volume provisioner's -// service account: systemRoleNode and systemRolePVProvisioner. These are required for -// provisioner to get node information and create persistent volumes. -func createProvisionerClusterRoleBinding(config *localTestConfig) { - subjects := []rbacv1beta1.Subject{ - { - Kind: rbacv1beta1.ServiceAccountKind, - Name: testServiceAccount, - Namespace: config.ns, - }, - } - - pvBinding := rbacv1beta1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1beta1", - Kind: "ClusterRoleBinding", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: pvBindingName, - }, - RoleRef: rbacv1beta1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: systemRolePVProvisioner, - }, - Subjects: subjects, - } - nodeBinding := rbacv1beta1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1beta1", - Kind: "ClusterRoleBinding", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: nodeBindingName, - }, - RoleRef: rbacv1beta1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: systemRoleNode, - }, - Subjects: subjects, - } - - deleteClusterRoleBinding(config) - _, err := config.client.RbacV1beta1().ClusterRoleBindings().Create(&pvBinding) - Expect(err).NotTo(HaveOccurred()) - _, err = config.client.RbacV1beta1().ClusterRoleBindings().Create(&nodeBinding) - Expect(err).NotTo(HaveOccurred()) -} - -func deleteClusterRoleBinding(config *localTestConfig) { - // These role bindings are created in provisioner; we just ensure it's - // deleted and do not panic on error. - config.client.RbacV1beta1().ClusterRoleBindings().Delete(nodeBindingName, metav1.NewDeleteOptions(0)) - config.client.RbacV1beta1().ClusterRoleBindings().Delete(pvBindingName, metav1.NewDeleteOptions(0)) -} - -func createVolumeConfigMap(config *localTestConfig) { - // MountConfig and ProvisionerConfiguration from - // https://github.com/kubernetes-incubator/external-storage/blob/master/local-volume/provisioner/pkg/common/common.go - type MountConfig struct { - // The hostpath directory - HostDir string `json:"hostDir" yaml:"hostDir"` - MountDir string `json:"mountDir" yaml:"mountDir"` - } - type ProvisionerConfiguration struct { - // StorageClassConfig defines configuration of Provisioner's storage classes - StorageClassConfig map[string]MountConfig `json:"storageClassMap" yaml:"storageClassMap"` - } - var provisionerConfig ProvisionerConfiguration - provisionerConfig.StorageClassConfig = map[string]MountConfig{ - config.scName: { - HostDir: config.discoveryDir, - MountDir: provisionerDefaultMountRoot, - }, - } - - data, err := yaml.Marshal(&provisionerConfig.StorageClassConfig) - Expect(err).NotTo(HaveOccurred()) - - configMap := v1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: volumeConfigName, - Namespace: config.ns, - }, - Data: map[string]string{ - "storageClassMap": string(data), - }, - } - - _, err = config.client.CoreV1().ConfigMaps(config.ns).Create(&configMap) - Expect(err).NotTo(HaveOccurred()) -} - -func createProvisionerDaemonset(config *localTestConfig) { - provisionerPrivileged := true - mountProp := v1.MountPropagationHostToContainer - - provisioner := &appsv1.DaemonSet{ - TypeMeta: metav1.TypeMeta{ - Kind: "DaemonSet", - APIVersion: "apps/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: daemonSetName, - }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": "local-volume-provisioner"}, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": "local-volume-provisioner"}, - }, - Spec: v1.PodSpec{ - ServiceAccountName: testServiceAccount, - Containers: []v1.Container{ - { - Name: "provisioner", - Image: provisionerImageName, - ImagePullPolicy: "Always", - SecurityContext: &v1.SecurityContext{ - Privileged: &provisionerPrivileged, - }, - Env: []v1.EnvVar{ - { - Name: "MY_NODE_NAME", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - { - Name: "MY_NAMESPACE", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "metadata.namespace", - }, - }, - }, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: volumeConfigName, - MountPath: "/etc/provisioner/config/", - }, - { - Name: "local-disks", - MountPath: provisionerDefaultMountRoot, - MountPropagation: &mountProp, - }, - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: volumeConfigName, - VolumeSource: v1.VolumeSource{ - ConfigMap: &v1.ConfigMapVolumeSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: volumeConfigName, - }, - }, - }, - }, - { - Name: "local-disks", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: config.discoveryDir, - }, - }, - }, - }, - }, - }, - }, - } - _, err := config.client.AppsV1().DaemonSets(config.ns).Create(provisioner) - Expect(err).NotTo(HaveOccurred()) - - kind := schema.GroupKind{Group: "extensions", Kind: "DaemonSet"} - framework.WaitForControlledPodsRunning(config.client, config.ns, daemonSetName, kind) -} - -func findProvisionerDaemonsetPodName(config *localTestConfig) string { - podList, err := config.client.CoreV1().Pods(config.ns).List(metav1.ListOptions{}) - if err != nil { - framework.Failf("could not get the pod list: %v", err) - return "" - } - pods := podList.Items - for _, pod := range pods { - if strings.HasPrefix(pod.Name, daemonSetName) && pod.Spec.NodeName == config.node0.Name { - return pod.Name - } - } - framework.Failf("Unable to find provisioner daemonset pod on node0") - return "" -} - -func deleteProvisionerDaemonset(config *localTestConfig) { - ds, err := config.client.AppsV1().DaemonSets(config.ns).Get(daemonSetName, metav1.GetOptions{}) - if ds == nil { - return - } - - err = config.client.AppsV1().DaemonSets(config.ns).Delete(daemonSetName, nil) - Expect(err).NotTo(HaveOccurred()) - - err = wait.PollImmediate(time.Second, time.Minute, func() (bool, error) { - pods, err := config.client.CoreV1().Pods(config.ns).List(metav1.ListOptions{}) - if err != nil { - return false, err - } - - for _, pod := range pods.Items { - if metav1.IsControlledBy(&pod, ds) { - // DaemonSet pod still exists - return false, nil - } - } - - // All DaemonSet pods are deleted - return true, nil - }) - Expect(err).NotTo(HaveOccurred()) -} - -// newLocalClaim creates a new persistent volume claim. -func newLocalClaim(config *localTestConfig) *v1.PersistentVolumeClaim { - claim := v1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "local-pvc-", - Namespace: config.ns, - }, - Spec: v1.PersistentVolumeClaimSpec{ - StorageClassName: &config.scName, - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceName(v1.ResourceStorage): resource.MustParse(testRequestSize), - }, - }, - }, - } - - return &claim -} - // newLocalClaim creates a new persistent volume claim. func newLocalClaimWithName(config *localTestConfig, name string) *v1.PersistentVolumeClaim { claim := v1.PersistentVolumeClaim{ @@ -1751,50 +1122,6 @@ func newLocalClaimWithName(config *localTestConfig, name string) *v1.PersistentV return &claim } -// waitForLocalPersistentVolume waits a local persistent volume with 'volumePath' to be available. -func waitForLocalPersistentVolume(c clientset.Interface, volumePath string) (*v1.PersistentVolume, error) { - var pv *v1.PersistentVolume - - for start := time.Now(); time.Since(start) < 10*time.Minute && pv == nil; time.Sleep(5 * time.Second) { - pvs, err := c.CoreV1().PersistentVolumes().List(metav1.ListOptions{}) - if err != nil { - return nil, err - } - if len(pvs.Items) == 0 { - continue - } - for _, p := range pvs.Items { - if p.Spec.PersistentVolumeSource.Local == nil || p.Spec.PersistentVolumeSource.Local.Path != volumePath { - continue - } - if p.Status.Phase != v1.VolumeAvailable { - continue - } - pv = &p - break - } - } - if pv == nil { - return nil, fmt.Errorf("Timeout while waiting for local persistent volume with path %v to be available", volumePath) - } - return pv, nil -} - -// findLocalPersistentVolume finds persistent volume with 'spec.local.path' equals 'volumePath'. -func findLocalPersistentVolume(c clientset.Interface, volumePath string) (*v1.PersistentVolume, error) { - pvs, err := c.CoreV1().PersistentVolumes().List(metav1.ListOptions{}) - if err != nil { - return nil, err - } - for _, p := range pvs.Items { - if p.Spec.PersistentVolumeSource.Local != nil && p.Spec.PersistentVolumeSource.Local.Path == volumePath { - return &p, nil - } - } - // Doesn't exist, that's fine, it could be invoked by early cleanup - return nil, nil -} - func createStatefulSet(config *localTestConfig, ssReplicas int32, volumeCount int, anti, parallel bool) *appsv1.StatefulSet { mounts := []v1.VolumeMount{} claims := []v1.PersistentVolumeClaim{} From 513ae63bf6d08f87f6dfcd6160677722217f47f9 Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Wed, 23 Jan 2019 13:49:41 +0800 Subject: [PATCH 2/2] Add a new stress test to replace old provisioner stress test --- test/e2e/storage/BUILD | 1 + test/e2e/storage/persistent_volumes-local.go | 198 +++++++++++++++++++ 2 files changed, 199 insertions(+) diff --git a/test/e2e/storage/BUILD b/test/e2e/storage/BUILD index b18f6c52c685..34162b867357 100644 --- a/test/e2e/storage/BUILD +++ b/test/e2e/storage/BUILD @@ -60,6 +60,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/version:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", diff --git a/test/e2e/storage/persistent_volumes-local.go b/test/e2e/storage/persistent_volumes-local.go index 08d4cf040017..d6427b0b1d37 100644 --- a/test/e2e/storage/persistent_volumes-local.go +++ b/test/e2e/storage/persistent_volumes-local.go @@ -21,6 +21,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "time" . "github.com/onsi/ginkgo" @@ -29,12 +30,14 @@ import ( appsv1 "k8s.io/api/apps/v1" "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/utils" @@ -432,8 +435,203 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { }) }) + Context("Stress with local volumes [Serial]", func() { + var ( + allLocalVolumes = make(map[string][]*localTestVolume) + volType = TmpfsLocalVolumeType + stopCh = make(chan struct{}) + wg sync.WaitGroup + ) + + const ( + volsPerNode = 10 // Make this non-divisable by volsPerPod to increase changes of partial binding failure + volsPerPod = 3 + podsFactor = 4 + ) + + BeforeEach(func() { + setupStorageClass(config, &waitMode) + for _, node := range config.nodes { + By(fmt.Sprintf("Setting up %d local volumes on node %q", volsPerNode, node.Name)) + allLocalVolumes[node.Name] = setupLocalVolumes(config, volType, &node, volsPerNode) + } + By(fmt.Sprintf("Create %d PVs", volsPerNode*len(config.nodes))) + var err error + for _, localVolumes := range allLocalVolumes { + for _, localVolume := range localVolumes { + pvConfig := makeLocalPVConfig(config, localVolume) + localVolume.pv, err = framework.CreatePV(config.client, framework.MakePersistentVolume(pvConfig)) + framework.ExpectNoError(err) + } + } + By("Start a goroutine to recycle unbound PVs") + wg.Add(1) + go func() { + defer wg.Done() + w, err := config.client.CoreV1().PersistentVolumes().Watch(metav1.ListOptions{}) + framework.ExpectNoError(err) + if w == nil { + return + } + defer w.Stop() + for { + select { + case event := <-w.ResultChan(): + if event.Type != watch.Modified { + continue + } + pv, ok := event.Object.(*v1.PersistentVolume) + if !ok { + continue + } + if pv.Status.Phase == v1.VolumeBound || pv.Status.Phase == v1.VolumeAvailable { + continue + } + pv, err = config.client.CoreV1().PersistentVolumes().Get(pv.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + continue + } + // Delete and create a new PV for same local volume storage + By(fmt.Sprintf("Delete %q and create a new PV for same local volume storage", pv.Name)) + for _, localVolumes := range allLocalVolumes { + for _, localVolume := range localVolumes { + if localVolume.pv.Name != pv.Name { + continue + } + err = config.client.CoreV1().PersistentVolumes().Delete(pv.Name, &metav1.DeleteOptions{}) + framework.ExpectNoError(err) + pvConfig := makeLocalPVConfig(config, localVolume) + localVolume.pv, err = framework.CreatePV(config.client, framework.MakePersistentVolume(pvConfig)) + framework.ExpectNoError(err) + } + } + case <-stopCh: + return + } + } + }() + }) + + AfterEach(func() { + for nodeName, localVolumes := range allLocalVolumes { + By(fmt.Sprintf("Cleaning up %d local volumes on node %q", len(localVolumes), nodeName)) + cleanupLocalVolumes(config, localVolumes) + } + cleanupStorageClass(config) + By("Wait for recycle goroutine to finish") + close(stopCh) + wg.Wait() + }) + + It("should be able to process many pods and reuse local volumes", func() { + var ( + podsLock sync.Mutex + // Have one extra pod pending + numConcurrentPods = volsPerNode/volsPerPod*len(config.nodes) + 1 + totalPods = numConcurrentPods * podsFactor + numCreated = 0 + numFinished = 0 + pods = map[string]*v1.Pod{} + ) + + // Create pods gradually instead of all at once because scheduler has + // exponential backoff + By(fmt.Sprintf("Creating %v pods periodically", numConcurrentPods)) + stop := make(chan struct{}) + go wait.Until(func() { + podsLock.Lock() + defer podsLock.Unlock() + + if numCreated >= totalPods { + // Created all the pods for the test + return + } + + if len(pods) > numConcurrentPods/2 { + // Too many outstanding pods + return + } + + for i := 0; i < numConcurrentPods; i++ { + pvcs := []*v1.PersistentVolumeClaim{} + for j := 0; j < volsPerPod; j++ { + pvc := framework.MakePersistentVolumeClaim(makeLocalPVCConfig(config, volType), config.ns) + pvc, err := framework.CreatePVC(config.client, config.ns, pvc) + framework.ExpectNoError(err) + pvcs = append(pvcs, pvc) + } + + pod := framework.MakeSecPod(config.ns, pvcs, false, "sleep 1", false, false, selinuxLabel, nil) + pod, err := config.client.CoreV1().Pods(config.ns).Create(pod) + Expect(err).NotTo(HaveOccurred()) + pods[pod.Name] = pod + numCreated++ + } + }, 2*time.Second, stop) + + defer func() { + close(stop) + podsLock.Lock() + defer podsLock.Unlock() + + for _, pod := range pods { + if err := deletePodAndPVCs(config, pod); err != nil { + framework.Logf("Deleting pod %v failed: %v", pod.Name, err) + } + } + }() + + By("Waiting for all pods to complete successfully") + err := wait.PollImmediate(time.Second, 5*time.Minute, func() (done bool, err error) { + podsList, err := config.client.CoreV1().Pods(config.ns).List(metav1.ListOptions{}) + if err != nil { + return false, err + } + + podsLock.Lock() + defer podsLock.Unlock() + + for _, pod := range podsList.Items { + switch pod.Status.Phase { + case v1.PodSucceeded: + // Delete pod and its PVCs + if err := deletePodAndPVCs(config, &pod); err != nil { + return false, err + } + delete(pods, pod.Name) + numFinished++ + framework.Logf("%v/%v pods finished", numFinished, totalPods) + case v1.PodFailed: + case v1.PodUnknown: + return false, fmt.Errorf("pod %v is in %v phase", pod.Name, pod.Status.Phase) + } + } + + return numFinished == totalPods, nil + }) + Expect(err).ToNot(HaveOccurred()) + }) + }) }) +func deletePodAndPVCs(config *localTestConfig, pod *v1.Pod) error { + framework.Logf("Deleting pod %v", pod.Name) + if err := config.client.CoreV1().Pods(config.ns).Delete(pod.Name, nil); err != nil { + return err + } + + // Delete PVCs + for _, vol := range pod.Spec.Volumes { + pvcSource := vol.VolumeSource.PersistentVolumeClaim + if pvcSource != nil { + if err := framework.DeletePersistentVolumeClaim(config.client, pvcSource.ClaimName, config.ns); err != nil { + return err + } + } + } + return nil +} + type makeLocalPodWith func(config *localTestConfig, volume *localTestVolume, nodeName string) *v1.Pod func testPodWithNodeConflict(config *localTestConfig, testVolType localVolumeType, nodeName string, makeLocalPodFunc makeLocalPodWith, bindingMode storagev1.VolumeBindingMode) {