diff --git a/test/e2e/storage/BUILD b/test/e2e/storage/BUILD index e151e87e7371..34162b867357 100644 --- a/test/e2e/storage/BUILD +++ b/test/e2e/storage/BUILD @@ -60,6 +60,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/version:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", @@ -82,7 +83,6 @@ go_library( "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/github.com/prometheus/common/model:go_default_library", "//vendor/google.golang.org/api/googleapi:go_default_library", - "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/test/e2e/storage/persistent_volumes-local.go b/test/e2e/storage/persistent_volumes-local.go index 75f68ff506a0..d6427b0b1d37 100644 --- a/test/e2e/storage/persistent_volumes-local.go +++ b/test/e2e/storage/persistent_volumes-local.go @@ -18,7 +18,6 @@ package storage import ( "fmt" - "path" "path/filepath" "strconv" "strings" @@ -27,19 +26,18 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "sigs.k8s.io/yaml" appsv1 "k8s.io/api/apps/v1" "k8s.io/api/core/v1" - rbacv1beta1 "k8s.io/api/rbac/v1beta1" storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/utils" @@ -137,24 +135,6 @@ const ( testFileContent = "test-file-content" testSCPrefix = "local-volume-test-storageclass" - // Following are constants used for provisioner e2e tests. - // - // testServiceAccount is the service account for bootstrapper - testServiceAccount = "local-storage-admin" - // volumeConfigName is the configmap passed to bootstrapper and provisioner - volumeConfigName = "local-volume-config" - // provisioner image used for e2e tests - provisionerImageName = "quay.io/external_storage/local-volume-provisioner:v2.1.0" - // provisioner daemonSetName name - daemonSetName = "local-volume-provisioner" - // provisioner default mount point folder - provisionerDefaultMountRoot = "/mnt/local-storage" - // provisioner node/pv cluster role binding - nodeBindingName = "local-storage:provisioner-node-binding" - pvBindingName = "local-storage:provisioner-pv-binding" - systemRoleNode = "system:node" - systemRolePVProvisioner = "system:persistent-volume-provisioner" - // A sample request size testRequestSize = "10Mi" @@ -398,107 +378,6 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { }) }) - Context("Local volume provisioner [Serial]", func() { - var volumePath string - - BeforeEach(func() { - setupStorageClass(config, &immediateMode) - setupLocalVolumeProvisioner(config) - volumePath = path.Join(config.discoveryDir, fmt.Sprintf("vol-%v", string(uuid.NewUUID()))) - setupLocalVolumeProvisionerMountPoint(config, volumePath, config.node0) - }) - - AfterEach(func() { - cleanupLocalVolumeProvisionerMountPoint(config, volumePath, config.node0) - cleanupLocalVolumeProvisioner(config) - cleanupStorageClass(config) - }) - - It("should create and recreate local persistent volume", func() { - By("Starting a provisioner daemonset") - createProvisionerDaemonset(config) - - By("Waiting for a PersistentVolume to be created") - oldPV, err := waitForLocalPersistentVolume(config.client, volumePath) - Expect(err).NotTo(HaveOccurred()) - - // Create a persistent volume claim for local volume: the above volume will be bound. - By("Creating a persistent volume claim") - claim, err := config.client.CoreV1().PersistentVolumeClaims(config.ns).Create(newLocalClaim(config)) - Expect(err).NotTo(HaveOccurred()) - err = framework.WaitForPersistentVolumeClaimPhase( - v1.ClaimBound, config.client, claim.Namespace, claim.Name, framework.Poll, 1*time.Minute) - Expect(err).NotTo(HaveOccurred()) - - claim, err = config.client.CoreV1().PersistentVolumeClaims(config.ns).Get(claim.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - Expect(claim.Spec.VolumeName).To(Equal(oldPV.Name)) - - // Delete the persistent volume claim: file will be cleaned up and volume be re-created. - By("Deleting the persistent volume claim to clean up persistent volume and re-create one") - writeCmd := createWriteCmd(volumePath, testFile, testFileContent, DirectoryLocalVolumeType) - err = issueNodeCommand(config, writeCmd, config.node0) - Expect(err).NotTo(HaveOccurred()) - err = config.client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, &metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred()) - - By("Waiting for a new PersistentVolume to be re-created") - newPV, err := waitForLocalPersistentVolume(config.client, volumePath) - Expect(err).NotTo(HaveOccurred()) - Expect(newPV.UID).NotTo(Equal(oldPV.UID)) - fileDoesntExistCmd := createFileDoesntExistCmd(volumePath, testFile) - err = issueNodeCommand(config, fileDoesntExistCmd, config.node0) - Expect(err).NotTo(HaveOccurred()) - - By("Deleting provisioner daemonset") - deleteProvisionerDaemonset(config) - }) - It("should not create local persistent volume for filesystem volume that was not bind mounted", func() { - - directoryPath := filepath.Join(config.discoveryDir, "notbindmount") - By("Creating a directory, not bind mounted, in discovery directory") - mkdirCmd := fmt.Sprintf("mkdir -p %v -m 777", directoryPath) - err := issueNodeCommand(config, mkdirCmd, config.node0) - Expect(err).NotTo(HaveOccurred()) - - By("Starting a provisioner daemonset") - createProvisionerDaemonset(config) - - By("Allowing provisioner to run for 30s and discover potential local PVs") - time.Sleep(30 * time.Second) - - By("Examining provisioner logs for not an actual mountpoint message") - provisionerPodName := findProvisionerDaemonsetPodName(config) - logs, err := framework.GetPodLogs(config.client, config.ns, provisionerPodName, "" /*containerName*/) - Expect(err).NotTo(HaveOccurred(), - "Error getting logs from pod %s in namespace %s", provisionerPodName, config.ns) - - expectedLogMessage := "Path \"/mnt/local-storage/notbindmount\" is not an actual mountpoint" - Expect(strings.Contains(logs, expectedLogMessage)).To(BeTrue()) - - By("Deleting provisioner daemonset") - deleteProvisionerDaemonset(config) - }) - It("should discover dynamically created local persistent volume mountpoint in discovery directory", func() { - By("Starting a provisioner daemonset") - createProvisionerDaemonset(config) - - By("Creating a volume in discovery directory") - dynamicVolumePath := path.Join(config.discoveryDir, fmt.Sprintf("vol-%v", string(uuid.NewUUID()))) - setupLocalVolumeProvisionerMountPoint(config, dynamicVolumePath, config.node0) - - By("Waiting for the PersistentVolume to be created") - _, err := waitForLocalPersistentVolume(config.client, dynamicVolumePath) - Expect(err).NotTo(HaveOccurred()) - - By("Deleting provisioner daemonset") - deleteProvisionerDaemonset(config) - - By("Deleting volume in discovery directory") - cleanupLocalVolumeProvisionerMountPoint(config, dynamicVolumePath, config.node0) - }) - }) - Context("StatefulSet with pod affinity [Slow]", func() { var testVols map[string][]*localTestVolume const ( @@ -556,8 +435,13 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { }) }) - Context("Stress with local volume provisioner [Serial]", func() { - var testVols [][]string + Context("Stress with local volumes [Serial]", func() { + var ( + allLocalVolumes = make(map[string][]*localTestVolume) + volType = TmpfsLocalVolumeType + stopCh = make(chan struct{}) + wg sync.WaitGroup + ) const ( volsPerNode = 10 // Make this non-divisable by volsPerPod to increase changes of partial binding failure @@ -567,38 +451,79 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { BeforeEach(func() { setupStorageClass(config, &waitMode) - setupLocalVolumeProvisioner(config) - - testVols = [][]string{} - for i, node := range config.nodes { - By(fmt.Sprintf("Setting up local volumes on node %q", node.Name)) - paths := []string{} - for j := 0; j < volsPerNode; j++ { - volumePath := path.Join(config.discoveryDir, fmt.Sprintf("vol-%v", string(uuid.NewUUID()))) - setupLocalVolumeProvisionerMountPoint(config, volumePath, &config.nodes[i]) - paths = append(paths, volumePath) + for _, node := range config.nodes { + By(fmt.Sprintf("Setting up %d local volumes on node %q", volsPerNode, node.Name)) + allLocalVolumes[node.Name] = setupLocalVolumes(config, volType, &node, volsPerNode) + } + By(fmt.Sprintf("Create %d PVs", volsPerNode*len(config.nodes))) + var err error + for _, localVolumes := range allLocalVolumes { + for _, localVolume := range localVolumes { + pvConfig := makeLocalPVConfig(config, localVolume) + localVolume.pv, err = framework.CreatePV(config.client, framework.MakePersistentVolume(pvConfig)) + framework.ExpectNoError(err) } - testVols = append(testVols, paths) } - - By("Starting the local volume provisioner") - createProvisionerDaemonset(config) + By("Start a goroutine to recycle unbound PVs") + wg.Add(1) + go func() { + defer wg.Done() + w, err := config.client.CoreV1().PersistentVolumes().Watch(metav1.ListOptions{}) + framework.ExpectNoError(err) + if w == nil { + return + } + defer w.Stop() + for { + select { + case event := <-w.ResultChan(): + if event.Type != watch.Modified { + continue + } + pv, ok := event.Object.(*v1.PersistentVolume) + if !ok { + continue + } + if pv.Status.Phase == v1.VolumeBound || pv.Status.Phase == v1.VolumeAvailable { + continue + } + pv, err = config.client.CoreV1().PersistentVolumes().Get(pv.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + continue + } + // Delete and create a new PV for same local volume storage + By(fmt.Sprintf("Delete %q and create a new PV for same local volume storage", pv.Name)) + for _, localVolumes := range allLocalVolumes { + for _, localVolume := range localVolumes { + if localVolume.pv.Name != pv.Name { + continue + } + err = config.client.CoreV1().PersistentVolumes().Delete(pv.Name, &metav1.DeleteOptions{}) + framework.ExpectNoError(err) + pvConfig := makeLocalPVConfig(config, localVolume) + localVolume.pv, err = framework.CreatePV(config.client, framework.MakePersistentVolume(pvConfig)) + framework.ExpectNoError(err) + } + } + case <-stopCh: + return + } + } + }() }) AfterEach(func() { - By("Deleting provisioner daemonset") - deleteProvisionerDaemonset(config) - - for i, paths := range testVols { - for _, volumePath := range paths { - cleanupLocalVolumeProvisionerMountPoint(config, volumePath, &config.nodes[i]) - } + for nodeName, localVolumes := range allLocalVolumes { + By(fmt.Sprintf("Cleaning up %d local volumes on node %q", len(localVolumes), nodeName)) + cleanupLocalVolumes(config, localVolumes) } - cleanupLocalVolumeProvisioner(config) cleanupStorageClass(config) + By("Wait for recycle goroutine to finish") + close(stopCh) + wg.Wait() }) - It("should use be able to process many pods and reuse local volumes", func() { + It("should be able to process many pods and reuse local volumes", func() { var ( podsLock sync.Mutex // Have one extra pod pending @@ -611,7 +536,6 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { // Create pods gradually instead of all at once because scheduler has // exponential backoff - // TODO: this is still a bit slow because of the provisioner polling period By(fmt.Sprintf("Creating %v pods periodically", numConcurrentPods)) stop := make(chan struct{}) go wait.Until(func() { @@ -631,7 +555,7 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { for i := 0; i < numConcurrentPods; i++ { pvcs := []*v1.PersistentVolumeClaim{} for j := 0; j < volsPerPod; j++ { - pvc := framework.MakePersistentVolumeClaim(makeLocalPVCConfig(config, DirectoryLocalVolumeType), config.ns) + pvc := framework.MakePersistentVolumeClaim(makeLocalPVCConfig(config, volType), config.ns) pvc, err := framework.CreatePVC(config.client, config.ns, pvc) framework.ExpectNoError(err) pvcs = append(pvcs, pvc) @@ -1226,18 +1150,6 @@ func createLocalPVCsPVs(config *localTestConfig, volumes []*localTestVolume, mod } } -func makeLocalPod(config *localTestConfig, volume *localTestVolume, cmd string) *v1.Pod { - pod := framework.MakeSecPod(config.ns, []*v1.PersistentVolumeClaim{volume.pvc}, false, cmd, false, false, selinuxLabel, nil) - if pod == nil { - return pod - } - if volume.localVolumeType == BlockLocalVolumeType { - // Block e2e tests require utilities for writing to block devices (e.g. dd), and nginx has this utilities. - pod.Spec.Containers[0].Image = imageutils.GetE2EImage(imageutils.Nginx) - } - return pod -} - func makeLocalPodWithNodeAffinity(config *localTestConfig, volume *localTestVolume, nodeName string) (pod *v1.Pod) { pod = framework.MakeSecPod(config.ns, []*v1.PersistentVolumeClaim{volume.pvc}, false, "", false, false, selinuxLabel, nil) if pod == nil { @@ -1285,16 +1197,6 @@ func makeLocalPodWithNodeName(config *localTestConfig, volume *localTestVolume, return } -// createSecPod should be used when Pod requires non default SELinux labels -func createSecPod(config *localTestConfig, volume *localTestVolume, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions) (*v1.Pod, error) { - pod, err := framework.CreateSecPod(config.client, config.ns, []*v1.PersistentVolumeClaim{volume.pvc}, false, "", hostIPC, hostPID, seLinuxLabel, nil, framework.PodStartShortTimeout) - podNodeName, podNodeNameErr := podNodeName(config, pod) - Expect(podNodeNameErr).NotTo(HaveOccurred()) - framework.Logf("Security Context POD %q created on Node %q", pod.Name, podNodeName) - Expect(podNodeName).To(Equal(config.node0.Name)) - return pod, err -} - func createLocalPod(config *localTestConfig, volume *localTestVolume, fsGroup *int64) (*v1.Pod, error) { By("Creating a pod") return framework.CreateSecPod(config.client, config.ns, []*v1.PersistentVolumeClaim{volume.pvc}, false, "", false, false, selinuxLabel, fsGroup, framework.PodStartShortTimeout) @@ -1368,13 +1270,6 @@ func testReadFileContent(testFileDir string, testFile string, testFileContent st Expect(readOut).To(ContainSubstring(testFileContent)) } -// Create command to verify that the file doesn't exist -// to be executed via hostexec Pod on the node with the local PV -func createFileDoesntExistCmd(testFileDir string, testFile string) string { - testFilePath := filepath.Join(testFileDir, testFile) - return fmt.Sprintf("[ ! -e %s ]", testFilePath) -} - // Execute a read or write command in a pod. // Fail on error func podRWCmdExec(pod *v1.Pod, cmd string) string { @@ -1402,332 +1297,6 @@ func setupLocalVolumesPVCsPVs( return testVols } -func setupLocalVolumeProvisioner(config *localTestConfig) { - By("Bootstrapping local volume provisioner") - createServiceAccount(config) - createProvisionerClusterRoleBinding(config) - utils.PrivilegedTestPSPClusterRoleBinding(config.client, config.ns, false /* teardown */, []string{testServiceAccount}) - createVolumeConfigMap(config) - - for _, node := range config.nodes { - By(fmt.Sprintf("Initializing local volume discovery base path on node %v", node.Name)) - mkdirCmd := fmt.Sprintf("mkdir -p %v -m 777", config.discoveryDir) - err := issueNodeCommand(config, mkdirCmd, &node) - Expect(err).NotTo(HaveOccurred()) - } -} - -func cleanupLocalVolumeProvisioner(config *localTestConfig) { - By("Cleaning up cluster role binding") - deleteClusterRoleBinding(config) - utils.PrivilegedTestPSPClusterRoleBinding(config.client, config.ns, true /* teardown */, []string{testServiceAccount}) - - for _, node := range config.nodes { - By(fmt.Sprintf("Removing the test discovery directory on node %v", node.Name)) - removeCmd := fmt.Sprintf("[ ! -e %v ] || rm -r %v", config.discoveryDir, config.discoveryDir) - err := issueNodeCommand(config, removeCmd, &node) - Expect(err).NotTo(HaveOccurred()) - } -} - -func setupLocalVolumeProvisionerMountPoint(config *localTestConfig, volumePath string, node *v1.Node) { - By(fmt.Sprintf("Creating local directory at path %q", volumePath)) - mkdirCmd := fmt.Sprintf("mkdir %v -m 777", volumePath) - err := issueNodeCommand(config, mkdirCmd, node) - Expect(err).NotTo(HaveOccurred()) - - By(fmt.Sprintf("Mounting local directory at path %q", volumePath)) - mntCmd := fmt.Sprintf("sudo mount --bind %v %v", volumePath, volumePath) - err = issueNodeCommand(config, mntCmd, node) - Expect(err).NotTo(HaveOccurred()) -} - -func cleanupLocalVolumeProvisionerMountPoint(config *localTestConfig, volumePath string, node *v1.Node) { - By(fmt.Sprintf("Unmounting the test mount point from %q", volumePath)) - umountCmd := fmt.Sprintf("[ ! -e %v ] || sudo umount %v", volumePath, volumePath) - err := issueNodeCommand(config, umountCmd, node) - Expect(err).NotTo(HaveOccurred()) - - By("Removing the test mount point") - removeCmd := fmt.Sprintf("[ ! -e %v ] || rm -r %v", volumePath, volumePath) - err = issueNodeCommand(config, removeCmd, node) - Expect(err).NotTo(HaveOccurred()) - - By("Cleaning up persistent volume") - pv, err := findLocalPersistentVolume(config.client, volumePath) - Expect(err).NotTo(HaveOccurred()) - if pv != nil { - err = config.client.CoreV1().PersistentVolumes().Delete(pv.Name, &metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred()) - } -} - -func createServiceAccount(config *localTestConfig) { - serviceAccount := v1.ServiceAccount{ - TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ServiceAccount"}, - ObjectMeta: metav1.ObjectMeta{Name: testServiceAccount, Namespace: config.ns}, - } - _, err := config.client.CoreV1().ServiceAccounts(config.ns).Create(&serviceAccount) - Expect(err).NotTo(HaveOccurred()) -} - -// createProvisionerClusterRoleBinding creates two cluster role bindings for local volume provisioner's -// service account: systemRoleNode and systemRolePVProvisioner. These are required for -// provisioner to get node information and create persistent volumes. -func createProvisionerClusterRoleBinding(config *localTestConfig) { - subjects := []rbacv1beta1.Subject{ - { - Kind: rbacv1beta1.ServiceAccountKind, - Name: testServiceAccount, - Namespace: config.ns, - }, - } - - pvBinding := rbacv1beta1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1beta1", - Kind: "ClusterRoleBinding", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: pvBindingName, - }, - RoleRef: rbacv1beta1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: systemRolePVProvisioner, - }, - Subjects: subjects, - } - nodeBinding := rbacv1beta1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1beta1", - Kind: "ClusterRoleBinding", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: nodeBindingName, - }, - RoleRef: rbacv1beta1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: systemRoleNode, - }, - Subjects: subjects, - } - - deleteClusterRoleBinding(config) - _, err := config.client.RbacV1beta1().ClusterRoleBindings().Create(&pvBinding) - Expect(err).NotTo(HaveOccurred()) - _, err = config.client.RbacV1beta1().ClusterRoleBindings().Create(&nodeBinding) - Expect(err).NotTo(HaveOccurred()) -} - -func deleteClusterRoleBinding(config *localTestConfig) { - // These role bindings are created in provisioner; we just ensure it's - // deleted and do not panic on error. - config.client.RbacV1beta1().ClusterRoleBindings().Delete(nodeBindingName, metav1.NewDeleteOptions(0)) - config.client.RbacV1beta1().ClusterRoleBindings().Delete(pvBindingName, metav1.NewDeleteOptions(0)) -} - -func createVolumeConfigMap(config *localTestConfig) { - // MountConfig and ProvisionerConfiguration from - // https://github.com/kubernetes-incubator/external-storage/blob/master/local-volume/provisioner/pkg/common/common.go - type MountConfig struct { - // The hostpath directory - HostDir string `json:"hostDir" yaml:"hostDir"` - MountDir string `json:"mountDir" yaml:"mountDir"` - } - type ProvisionerConfiguration struct { - // StorageClassConfig defines configuration of Provisioner's storage classes - StorageClassConfig map[string]MountConfig `json:"storageClassMap" yaml:"storageClassMap"` - } - var provisionerConfig ProvisionerConfiguration - provisionerConfig.StorageClassConfig = map[string]MountConfig{ - config.scName: { - HostDir: config.discoveryDir, - MountDir: provisionerDefaultMountRoot, - }, - } - - data, err := yaml.Marshal(&provisionerConfig.StorageClassConfig) - Expect(err).NotTo(HaveOccurred()) - - configMap := v1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: volumeConfigName, - Namespace: config.ns, - }, - Data: map[string]string{ - "storageClassMap": string(data), - }, - } - - _, err = config.client.CoreV1().ConfigMaps(config.ns).Create(&configMap) - Expect(err).NotTo(HaveOccurred()) -} - -func createProvisionerDaemonset(config *localTestConfig) { - provisionerPrivileged := true - mountProp := v1.MountPropagationHostToContainer - - provisioner := &appsv1.DaemonSet{ - TypeMeta: metav1.TypeMeta{ - Kind: "DaemonSet", - APIVersion: "apps/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: daemonSetName, - }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": "local-volume-provisioner"}, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": "local-volume-provisioner"}, - }, - Spec: v1.PodSpec{ - ServiceAccountName: testServiceAccount, - Containers: []v1.Container{ - { - Name: "provisioner", - Image: provisionerImageName, - ImagePullPolicy: "Always", - SecurityContext: &v1.SecurityContext{ - Privileged: &provisionerPrivileged, - }, - Env: []v1.EnvVar{ - { - Name: "MY_NODE_NAME", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - { - Name: "MY_NAMESPACE", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "metadata.namespace", - }, - }, - }, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: volumeConfigName, - MountPath: "/etc/provisioner/config/", - }, - { - Name: "local-disks", - MountPath: provisionerDefaultMountRoot, - MountPropagation: &mountProp, - }, - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: volumeConfigName, - VolumeSource: v1.VolumeSource{ - ConfigMap: &v1.ConfigMapVolumeSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: volumeConfigName, - }, - }, - }, - }, - { - Name: "local-disks", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: config.discoveryDir, - }, - }, - }, - }, - }, - }, - }, - } - _, err := config.client.AppsV1().DaemonSets(config.ns).Create(provisioner) - Expect(err).NotTo(HaveOccurred()) - - kind := schema.GroupKind{Group: "extensions", Kind: "DaemonSet"} - framework.WaitForControlledPodsRunning(config.client, config.ns, daemonSetName, kind) -} - -func findProvisionerDaemonsetPodName(config *localTestConfig) string { - podList, err := config.client.CoreV1().Pods(config.ns).List(metav1.ListOptions{}) - if err != nil { - framework.Failf("could not get the pod list: %v", err) - return "" - } - pods := podList.Items - for _, pod := range pods { - if strings.HasPrefix(pod.Name, daemonSetName) && pod.Spec.NodeName == config.node0.Name { - return pod.Name - } - } - framework.Failf("Unable to find provisioner daemonset pod on node0") - return "" -} - -func deleteProvisionerDaemonset(config *localTestConfig) { - ds, err := config.client.AppsV1().DaemonSets(config.ns).Get(daemonSetName, metav1.GetOptions{}) - if ds == nil { - return - } - - err = config.client.AppsV1().DaemonSets(config.ns).Delete(daemonSetName, nil) - Expect(err).NotTo(HaveOccurred()) - - err = wait.PollImmediate(time.Second, time.Minute, func() (bool, error) { - pods, err := config.client.CoreV1().Pods(config.ns).List(metav1.ListOptions{}) - if err != nil { - return false, err - } - - for _, pod := range pods.Items { - if metav1.IsControlledBy(&pod, ds) { - // DaemonSet pod still exists - return false, nil - } - } - - // All DaemonSet pods are deleted - return true, nil - }) - Expect(err).NotTo(HaveOccurred()) -} - -// newLocalClaim creates a new persistent volume claim. -func newLocalClaim(config *localTestConfig) *v1.PersistentVolumeClaim { - claim := v1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "local-pvc-", - Namespace: config.ns, - }, - Spec: v1.PersistentVolumeClaimSpec{ - StorageClassName: &config.scName, - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceName(v1.ResourceStorage): resource.MustParse(testRequestSize), - }, - }, - }, - } - - return &claim -} - // newLocalClaim creates a new persistent volume claim. func newLocalClaimWithName(config *localTestConfig, name string) *v1.PersistentVolumeClaim { claim := v1.PersistentVolumeClaim{ @@ -1751,50 +1320,6 @@ func newLocalClaimWithName(config *localTestConfig, name string) *v1.PersistentV return &claim } -// waitForLocalPersistentVolume waits a local persistent volume with 'volumePath' to be available. -func waitForLocalPersistentVolume(c clientset.Interface, volumePath string) (*v1.PersistentVolume, error) { - var pv *v1.PersistentVolume - - for start := time.Now(); time.Since(start) < 10*time.Minute && pv == nil; time.Sleep(5 * time.Second) { - pvs, err := c.CoreV1().PersistentVolumes().List(metav1.ListOptions{}) - if err != nil { - return nil, err - } - if len(pvs.Items) == 0 { - continue - } - for _, p := range pvs.Items { - if p.Spec.PersistentVolumeSource.Local == nil || p.Spec.PersistentVolumeSource.Local.Path != volumePath { - continue - } - if p.Status.Phase != v1.VolumeAvailable { - continue - } - pv = &p - break - } - } - if pv == nil { - return nil, fmt.Errorf("Timeout while waiting for local persistent volume with path %v to be available", volumePath) - } - return pv, nil -} - -// findLocalPersistentVolume finds persistent volume with 'spec.local.path' equals 'volumePath'. -func findLocalPersistentVolume(c clientset.Interface, volumePath string) (*v1.PersistentVolume, error) { - pvs, err := c.CoreV1().PersistentVolumes().List(metav1.ListOptions{}) - if err != nil { - return nil, err - } - for _, p := range pvs.Items { - if p.Spec.PersistentVolumeSource.Local != nil && p.Spec.PersistentVolumeSource.Local.Path == volumePath { - return &p, nil - } - } - // Doesn't exist, that's fine, it could be invoked by early cleanup - return nil, nil -} - func createStatefulSet(config *localTestConfig, ssReplicas int32, volumeCount int, anti, parallel bool) *appsv1.StatefulSet { mounts := []v1.VolumeMount{} claims := []v1.PersistentVolumeClaim{}