From 6795820cfacd987045b6abfcb2691eb2d72bd3a8 Mon Sep 17 00:00:00 2001 From: carlory Date: Fri, 26 Apr 2024 10:55:32 +0800 Subject: [PATCH] remove the support for in-tree volume plugin cephfs --- cmd/kubelet/app/plugins.go | 2 - pkg/kubemark/hollow_kubelet.go | 2 - pkg/volume/cephfs/OWNERS | 15 - pkg/volume/cephfs/cephfs.go | 459 ---------------------------- pkg/volume/cephfs/cephfs_test.go | 251 --------------- pkg/volume/cephfs/doc.go | 19 -- test/e2e/storage/drivers/in_tree.go | 152 --------- test/e2e/storage/in_tree_volumes.go | 1 - test/utils/image/manifest.go | 3 - 9 files changed, 904 deletions(-) delete mode 100644 pkg/volume/cephfs/OWNERS delete mode 100644 pkg/volume/cephfs/cephfs.go delete mode 100644 pkg/volume/cephfs/cephfs_test.go delete mode 100644 pkg/volume/cephfs/doc.go diff --git a/cmd/kubelet/app/plugins.go b/cmd/kubelet/app/plugins.go index 4ec7a295a85d..da907b17a575 100644 --- a/cmd/kubelet/app/plugins.go +++ b/cmd/kubelet/app/plugins.go @@ -23,7 +23,6 @@ import ( // Volume plugins "k8s.io/kubernetes/pkg/volume" - "k8s.io/kubernetes/pkg/volume/cephfs" "k8s.io/kubernetes/pkg/volume/configmap" "k8s.io/kubernetes/pkg/volume/csi" "k8s.io/kubernetes/pkg/volume/downwardapi" @@ -63,7 +62,6 @@ func ProbeVolumePlugins(featureGate featuregate.FeatureGate) ([]volume.VolumePlu allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(volume.VolumeConfig{})...) allPlugins = append(allPlugins, secret.ProbeVolumePlugins()...) allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, cephfs.ProbeVolumePlugins()...) allPlugins = append(allPlugins, downwardapi.ProbeVolumePlugins()...) allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...) allPlugins = append(allPlugins, configmap.ProbeVolumePlugins()...) diff --git a/pkg/kubemark/hollow_kubelet.go b/pkg/kubemark/hollow_kubelet.go index fa4c71af36c5..100ab234b8b3 100644 --- a/pkg/kubemark/hollow_kubelet.go +++ b/pkg/kubemark/hollow_kubelet.go @@ -42,7 +42,6 @@ import ( kubeletutil "k8s.io/kubernetes/pkg/kubelet/util" "k8s.io/kubernetes/pkg/util/oom" "k8s.io/kubernetes/pkg/volume" - "k8s.io/kubernetes/pkg/volume/cephfs" "k8s.io/kubernetes/pkg/volume/configmap" "k8s.io/kubernetes/pkg/volume/csi" "k8s.io/kubernetes/pkg/volume/downwardapi" @@ -75,7 +74,6 @@ func volumePlugins() []volume.VolumePlugin { allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(volume.VolumeConfig{})...) allPlugins = append(allPlugins, secret.ProbeVolumePlugins()...) allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, cephfs.ProbeVolumePlugins()...) allPlugins = append(allPlugins, downwardapi.ProbeVolumePlugins()...) allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...) allPlugins = append(allPlugins, configmap.ProbeVolumePlugins()...) diff --git a/pkg/volume/cephfs/OWNERS b/pkg/volume/cephfs/OWNERS deleted file mode 100644 index 2c23a57619ef..000000000000 --- a/pkg/volume/cephfs/OWNERS +++ /dev/null @@ -1,15 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -approvers: - - jsafrane - - humblec - - saad-ali -reviewers: - - saad-ali - - jsafrane - - jingxu97 - - msau42 - - cofyc - - humblec -emeritus_approvers: - - rootfs diff --git a/pkg/volume/cephfs/cephfs.go b/pkg/volume/cephfs/cephfs.go deleted file mode 100644 index e6243dba1203..000000000000 --- a/pkg/volume/cephfs/cephfs.go +++ /dev/null @@ -1,459 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cephfs - -import ( - "context" - "fmt" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" - - "k8s.io/klog/v2" - "k8s.io/mount-utils" - utilstrings "k8s.io/utils/strings" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/kubernetes/pkg/volume" - "k8s.io/kubernetes/pkg/volume/util" -) - -// ProbeVolumePlugins is the primary entrypoint for volume plugins. -func ProbeVolumePlugins() []volume.VolumePlugin { - return []volume.VolumePlugin{&cephfsPlugin{nil}} -} - -type cephfsPlugin struct { - host volume.VolumeHost -} - -var _ volume.VolumePlugin = &cephfsPlugin{} - -const ( - cephfsPluginName = "kubernetes.io/cephfs" -) - -func (plugin *cephfsPlugin) Init(host volume.VolumeHost) error { - plugin.host = host - return nil -} - -func (plugin *cephfsPlugin) GetPluginName() string { - return cephfsPluginName -} - -func (plugin *cephfsPlugin) GetVolumeName(spec *volume.Spec) (string, error) { - mon, _, _, _, _, err := getVolumeSource(spec) - if err != nil { - return "", err - } - - return fmt.Sprintf("%v", mon), nil -} - -func (plugin *cephfsPlugin) CanSupport(spec *volume.Spec) bool { - return (spec.Volume != nil && spec.Volume.CephFS != nil) || (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.CephFS != nil) -} - -func (plugin *cephfsPlugin) RequiresRemount(spec *volume.Spec) bool { - return false -} - -func (plugin *cephfsPlugin) SupportsMountOption() bool { - return true -} - -func (plugin *cephfsPlugin) SupportsBulkVolumeVerification() bool { - return false -} - -func (plugin *cephfsPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { - return false, nil -} - -func (plugin *cephfsPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { - return []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - v1.ReadOnlyMany, - v1.ReadWriteMany, - } -} - -func (plugin *cephfsPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { - secretName, secretNs, err := getSecretNameAndNamespace(spec, pod.Namespace) - if err != nil { - return nil, err - } - secret := "" - if len(secretName) > 0 && len(secretNs) > 0 { - // if secret is provideded, retrieve it - kubeClient := plugin.host.GetKubeClient() - if kubeClient == nil { - return nil, fmt.Errorf("cannot get kube client") - } - secrets, err := kubeClient.CoreV1().Secrets(secretNs).Get(context.TODO(), secretName, metav1.GetOptions{}) - if err != nil { - err = fmt.Errorf("couldn't get secret %v/%v err: %w", secretNs, secretName, err) - return nil, err - } - for name, data := range secrets.Data { - secret = string(data) - klog.V(4).Infof("found ceph secret info: %s", name) - } - } - return plugin.newMounterInternal(spec, pod.UID, plugin.host.GetMounter(plugin.GetPluginName()), secret) -} - -func (plugin *cephfsPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, mounter mount.Interface, secret string) (volume.Mounter, error) { - mon, path, id, secretFile, readOnly, err := getVolumeSource(spec) - if err != nil { - return nil, err - } - - if id == "" { - id = "admin" - } - if path == "" { - path = "/" - } - if !strings.HasPrefix(path, "/") { - path = "/" + path - } - - if secretFile == "" { - secretFile = "/etc/ceph/" + id + ".secret" - } - - return &cephfsMounter{ - cephfs: &cephfs{ - podUID: podUID, - volName: spec.Name(), - mon: mon, - path: path, - secret: secret, - id: id, - secretFile: secretFile, - readonly: readOnly, - mounter: mounter, - plugin: plugin, - mountOptions: util.MountOptionFromSpec(spec), - }, - }, nil -} - -func (plugin *cephfsPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { - return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter(plugin.GetPluginName())) -} - -func (plugin *cephfsPlugin) newUnmounterInternal(volName string, podUID types.UID, mounter mount.Interface) (volume.Unmounter, error) { - return &cephfsUnmounter{ - cephfs: &cephfs{ - podUID: podUID, - volName: volName, - mounter: mounter, - plugin: plugin}, - }, nil -} - -func (plugin *cephfsPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) { - cephfsVolume := &v1.Volume{ - Name: volumeName, - VolumeSource: v1.VolumeSource{ - CephFS: &v1.CephFSVolumeSource{ - Monitors: []string{}, - Path: mountPath, - }, - }, - } - return volume.ReconstructedVolume{ - Spec: volume.NewSpecFromVolume(cephfsVolume), - }, nil -} - -// CephFS volumes represent a bare host file or directory mount of an CephFS export. -type cephfs struct { - volName string - podUID types.UID - mon []string - path string - id string - secret string `datapolicy:"token"` - secretFile string - readonly bool - mounter mount.Interface - plugin *cephfsPlugin - volume.MetricsNil - mountOptions []string -} - -type cephfsMounter struct { - *cephfs -} - -var _ volume.Mounter = &cephfsMounter{} - -func (cephfsVolume *cephfsMounter) GetAttributes() volume.Attributes { - return volume.Attributes{ - ReadOnly: cephfsVolume.readonly, - Managed: false, - SELinuxRelabel: false, - } -} - -// SetUp attaches the disk and bind mounts to the volume path. -func (cephfsVolume *cephfsMounter) SetUp(mounterArgs volume.MounterArgs) error { - return cephfsVolume.SetUpAt(cephfsVolume.GetPath(), mounterArgs) -} - -// SetUpAt attaches the disk and bind mounts to the volume path. -func (cephfsVolume *cephfsMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { - notMnt, err := cephfsVolume.mounter.IsLikelyNotMountPoint(dir) - klog.V(4).Infof("CephFS mount set up: %s %v %v", dir, !notMnt, err) - if err != nil && !os.IsNotExist(err) { - return err - } - if !notMnt { - return nil - } - - if err := os.MkdirAll(dir, 0750); err != nil { - return err - } - - // check whether it belongs to fuse, if not, default to use kernel mount. - if cephfsVolume.checkFuseMount() { - klog.V(4).Info("CephFS fuse mount.") - err = cephfsVolume.execFuseMount(dir) - // cleanup no matter if fuse mount fail. - keyringPath := cephfsVolume.GetKeyringPath() - _, StatErr := os.Stat(keyringPath) - if !os.IsNotExist(StatErr) { - os.RemoveAll(keyringPath) - } - if err == nil { - // cephfs fuse mount succeeded. - return nil - } - // if cephfs fuse mount failed, fallback to kernel mount. - klog.V(2).Infof("CephFS fuse mount failed: %v, fallback to kernel mount.", err) - - } - klog.V(4).Info("CephFS kernel mount.") - - err = cephfsVolume.execMount(dir) - if err != nil { - // cleanup upon failure. - mount.CleanupMountPoint(dir, cephfsVolume.mounter, false) - return err - } - return nil -} - -type cephfsUnmounter struct { - *cephfs -} - -var _ volume.Unmounter = &cephfsUnmounter{} - -// TearDown unmounts the bind mount -func (cephfsVolume *cephfsUnmounter) TearDown() error { - return cephfsVolume.TearDownAt(cephfsVolume.GetPath()) -} - -// TearDownAt unmounts the bind mount -func (cephfsVolume *cephfsUnmounter) TearDownAt(dir string) error { - return mount.CleanupMountPoint(dir, cephfsVolume.mounter, false) -} - -// GetPath creates global mount path -func (cephfsVolume *cephfs) GetPath() string { - name := cephfsPluginName - return cephfsVolume.plugin.host.GetPodVolumeDir(cephfsVolume.podUID, utilstrings.EscapeQualifiedName(name), cephfsVolume.volName) -} - -// GetKeyringPath creates cephfuse keyring path -func (cephfsVolume *cephfs) GetKeyringPath() string { - name := cephfsPluginName - volumeDir := cephfsVolume.plugin.host.GetPodVolumeDir(cephfsVolume.podUID, utilstrings.EscapeQualifiedName(name), cephfsVolume.volName) - volumeKeyringDir := volumeDir + "~keyring" - return volumeKeyringDir -} - -func (cephfsVolume *cephfs) execMount(mountpoint string) error { - // cephfs mount option - cephSensitiveOpt := []string{"name=" + cephfsVolume.id} - // override secretfile if secret is provided - if cephfsVolume.secret != "" { - cephSensitiveOpt = append(cephSensitiveOpt, "secret="+cephfsVolume.secret) - } else { - cephSensitiveOpt = append(cephSensitiveOpt, "secretfile="+cephfsVolume.secretFile) - } - // build option array - opt := []string{} - if cephfsVolume.readonly { - opt = append(opt, "ro") - } - - // build src like mon1:6789,mon2:6789,mon3:6789:/ - src := strings.Join(cephfsVolume.mon, ",") + ":" + cephfsVolume.path - - opt = util.JoinMountOptions(cephfsVolume.mountOptions, opt) - if err := cephfsVolume.mounter.MountSensitive(src, mountpoint, "ceph", opt, cephSensitiveOpt); err != nil { - return fmt.Errorf("CephFS: mount failed: %v", err) - } - - return nil -} - -func (cephfsVolume *cephfsMounter) checkFuseMount() bool { - execute := cephfsVolume.plugin.host.GetExec(cephfsVolume.plugin.GetPluginName()) - switch runtime.GOOS { - case "linux": - if _, err := execute.Command("/usr/bin/test", "-x", "/sbin/mount.fuse.ceph").CombinedOutput(); err == nil { - klog.V(4).Info("/sbin/mount.fuse.ceph exists, it should be fuse mount.") - return true - } - return false - } - return false -} - -func (cephfsVolume *cephfs) execFuseMount(mountpoint string) error { - // cephfs keyring file - keyringFile := "" - // override secretfile if secret is provided - if cephfsVolume.secret != "" { - // TODO: cephfs fuse currently doesn't support secret option, - // remove keyring file create once secret option is supported. - klog.V(4).Info("cephfs mount begin using fuse.") - - keyringPath := cephfsVolume.GetKeyringPath() - if err := os.MkdirAll(keyringPath, 0750); err != nil { - return err - } - - payload := make(map[string]util.FileProjection, 1) - var fileProjection util.FileProjection - - keyring := fmt.Sprintf("[client.%s]\nkey = %s\n", cephfsVolume.id, cephfsVolume.secret) - - fileProjection.Data = []byte(keyring) - fileProjection.Mode = int32(0644) - fileName := cephfsVolume.id + ".keyring" - - payload[fileName] = fileProjection - - writerContext := fmt.Sprintf("cephfuse:%v.keyring", cephfsVolume.id) - writer, err := util.NewAtomicWriter(keyringPath, writerContext) - if err != nil { - klog.Errorf("failed to create atomic writer: %v", err) - return err - } - - err = writer.Write(payload, nil /*setPerms*/) - if err != nil { - klog.Errorf("failed to write payload to dir: %v", err) - return err - } - - keyringFile = filepath.Join(keyringPath, fileName) - - } else { - keyringFile = cephfsVolume.secretFile - } - // build src like mon1:6789,mon2:6789,mon3:6789:/ - src := strings.Join(cephfsVolume.mon, ",") - - mountArgs := []string{} - mountArgs = append(mountArgs, "-k") - mountArgs = append(mountArgs, keyringFile) - mountArgs = append(mountArgs, "-m") - mountArgs = append(mountArgs, src) - mountArgs = append(mountArgs, mountpoint) - mountArgs = append(mountArgs, "-r") - mountArgs = append(mountArgs, cephfsVolume.path) - mountArgs = append(mountArgs, "--id") - mountArgs = append(mountArgs, cephfsVolume.id) - - // build option array - opt := []string{} - if cephfsVolume.readonly { - opt = append(opt, "ro") - } - opt = util.JoinMountOptions(cephfsVolume.mountOptions, opt) - if len(opt) > 0 { - mountArgs = append(mountArgs, "-o") - mountArgs = append(mountArgs, strings.Join(opt, ",")) - } - - klog.V(4).Infof("Mounting cmd ceph-fuse with arguments (%s)", mountArgs) - command := exec.Command("ceph-fuse", mountArgs...) - output, err := command.CombinedOutput() - if err != nil || !(strings.Contains(string(output), "starting fuse")) { - return fmt.Errorf("Ceph-fuse failed: %v\narguments: %s\nOutput: %s", err, mountArgs, string(output)) - } - - return nil -} - -func getVolumeSource(spec *volume.Spec) ([]string, string, string, string, bool, error) { - if spec.Volume != nil && spec.Volume.CephFS != nil { - mon := spec.Volume.CephFS.Monitors - path := spec.Volume.CephFS.Path - user := spec.Volume.CephFS.User - secretFile := spec.Volume.CephFS.SecretFile - readOnly := spec.Volume.CephFS.ReadOnly - return mon, path, user, secretFile, readOnly, nil - } else if spec.PersistentVolume != nil && - spec.PersistentVolume.Spec.CephFS != nil { - mon := spec.PersistentVolume.Spec.CephFS.Monitors - path := spec.PersistentVolume.Spec.CephFS.Path - user := spec.PersistentVolume.Spec.CephFS.User - secretFile := spec.PersistentVolume.Spec.CephFS.SecretFile - readOnly := spec.PersistentVolume.Spec.CephFS.ReadOnly - return mon, path, user, secretFile, readOnly, nil - } - - return nil, "", "", "", false, fmt.Errorf("Spec does not reference a CephFS volume type") -} - -func getSecretNameAndNamespace(spec *volume.Spec, defaultNamespace string) (string, string, error) { - if spec.Volume != nil && spec.Volume.CephFS != nil { - localSecretRef := spec.Volume.CephFS.SecretRef - if localSecretRef != nil { - return localSecretRef.Name, defaultNamespace, nil - } - return "", "", nil - - } else if spec.PersistentVolume != nil && - spec.PersistentVolume.Spec.CephFS != nil { - secretRef := spec.PersistentVolume.Spec.CephFS.SecretRef - secretNs := defaultNamespace - if secretRef != nil { - if len(secretRef.Namespace) != 0 { - secretNs = secretRef.Namespace - } - return secretRef.Name, secretNs, nil - } - return "", "", nil - } - return "", "", fmt.Errorf("Spec does not reference an CephFS volume type") -} diff --git a/pkg/volume/cephfs/cephfs_test.go b/pkg/volume/cephfs/cephfs_test.go deleted file mode 100644 index 874395b60bac..000000000000 --- a/pkg/volume/cephfs/cephfs_test.go +++ /dev/null @@ -1,251 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cephfs - -import ( - "os" - "path/filepath" - "testing" - - "k8s.io/mount-utils" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - utiltesting "k8s.io/client-go/util/testing" - "k8s.io/kubernetes/pkg/volume" - volumetest "k8s.io/kubernetes/pkg/volume/testing" -) - -func TestCanSupport(t *testing.T) { - tmpDir, err := utiltesting.MkTmpdir("cephTest") - if err != nil { - t.Fatalf("can't make a temp dir: %v", err) - } - defer os.RemoveAll(tmpDir) - plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(t, tmpDir, nil, nil)) - plug, err := plugMgr.FindPluginByName("kubernetes.io/cephfs") - if err != nil { - t.Fatal("Can't find the plugin by name") - } - if plug.GetPluginName() != "kubernetes.io/cephfs" { - t.Errorf("Wrong name: %s", plug.GetPluginName()) - } - if plug.CanSupport(&volume.Spec{Volume: &v1.Volume{VolumeSource: v1.VolumeSource{}}}) { - t.Errorf("Expected false") - } - if !plug.CanSupport(&volume.Spec{Volume: &v1.Volume{VolumeSource: v1.VolumeSource{CephFS: &v1.CephFSVolumeSource{}}}}) { - t.Errorf("Expected true") - } -} - -func TestPlugin(t *testing.T) { - tmpDir, err := utiltesting.MkTmpdir("cephTest") - if err != nil { - t.Fatalf("can't make a temp dir: %v", err) - } - defer os.RemoveAll(tmpDir) - plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(t, tmpDir, nil, nil)) - plug, err := plugMgr.FindPluginByName("kubernetes.io/cephfs") - if err != nil { - t.Errorf("Can't find the plugin by name") - } - spec := &v1.Volume{ - Name: "vol1", - VolumeSource: v1.VolumeSource{ - CephFS: &v1.CephFSVolumeSource{ - Monitors: []string{"a", "b"}, - User: "user", - SecretRef: nil, - SecretFile: "/etc/ceph/user.secret", - }, - }, - } - mounter, err := plug.(*cephfsPlugin).newMounterInternal(volume.NewSpecFromVolume(spec), types.UID("poduid"), mount.NewFakeMounter(nil), "secrets") - if err != nil { - t.Errorf("Failed to make a new Mounter: %v", err) - } - if mounter == nil { - t.Errorf("Got a nil Mounter") - } - volumePath := mounter.GetPath() - volpath := filepath.Join(tmpDir, "pods/poduid/volumes/kubernetes.io~cephfs/vol1") - if volumePath != volpath { - t.Errorf("Got unexpected path: %s", volumePath) - } - if err := mounter.SetUp(volume.MounterArgs{}); err != nil { - t.Errorf("Expected success, got: %v", err) - } - if _, err := os.Stat(volumePath); err != nil { - if os.IsNotExist(err) { - t.Errorf("SetUp() failed, volume path not created: %s", volumePath) - } else { - t.Errorf("SetUp() failed: %v", err) - } - } - unmounter, err := plug.(*cephfsPlugin).newUnmounterInternal("vol1", types.UID("poduid"), mount.NewFakeMounter(nil)) - if err != nil { - t.Errorf("Failed to make a new Unmounter: %v", err) - } - if unmounter == nil { - t.Errorf("Got a nil Unmounter") - } - if err := unmounter.TearDown(); err != nil { - t.Errorf("Expected success, got: %v", err) - } - if _, err := os.Stat(volumePath); err == nil { - t.Errorf("TearDown() failed, volume path still exists: %s", volumePath) - } else if !os.IsNotExist(err) { - t.Errorf("TearDown() failed: %v", err) - } -} - -func TestConstructVolumeSpec(t *testing.T) { - tmpDir, err := utiltesting.MkTmpdir("cephTest") - if err != nil { - t.Fatalf("Can't make a temp dir: %v", err) - } - defer os.RemoveAll(tmpDir) - plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(t, tmpDir, nil, nil)) - plug, err := plugMgr.FindPluginByName("kubernetes.io/cephfs") - if err != nil { - t.Errorf("can't find cephfs plugin by name") - } - - cephfsVol, err := plug.(*cephfsPlugin).ConstructVolumeSpec("cephfsVolume", "/cephfsVolume/") - if err != nil { - t.Errorf("ConstructVolumeSpec() failed: %v", err) - } - - if cephfsVol.Spec.Name() != "cephfsVolume" { - t.Errorf("Get wrong cephfs spec name, got: %s", cephfsVol.Spec.Name()) - } -} - -type testcase struct { - name string - defaultNs string - spec *volume.Spec - // Expected return of the test - expectedName string - expectedNs string - expectedError error -} - -func TestGetSecretNameAndNamespaceForPV(t *testing.T) { - tests := []testcase{ - { - name: "persistent volume source", - defaultNs: "default", - spec: &volume.Spec{ - PersistentVolume: &v1.PersistentVolume{ - Spec: v1.PersistentVolumeSpec{ - PersistentVolumeSource: v1.PersistentVolumeSource{ - CephFS: &v1.CephFSPersistentVolumeSource{ - Monitors: []string{"a", "b"}, - User: "user", - SecretRef: &v1.SecretReference{ - Name: "name", - Namespace: "ns", - }, - SecretFile: "/etc/ceph/user.secret", - }, - }, - }, - }, - }, - expectedName: "name", - expectedNs: "ns", - expectedError: nil, - }, - { - name: "persistent volume source without namespace", - defaultNs: "default", - spec: &volume.Spec{ - PersistentVolume: &v1.PersistentVolume{ - Spec: v1.PersistentVolumeSpec{ - PersistentVolumeSource: v1.PersistentVolumeSource{ - CephFS: &v1.CephFSPersistentVolumeSource{ - Monitors: []string{"a", "b"}, - User: "user", - SecretRef: &v1.SecretReference{ - Name: "name", - }, - SecretFile: "/etc/ceph/user.secret", - }, - }, - }, - }, - }, - expectedName: "name", - expectedNs: "default", - expectedError: nil, - }, - { - name: "pod volume source", - defaultNs: "default", - spec: &volume.Spec{ - Volume: &v1.Volume{ - VolumeSource: v1.VolumeSource{ - CephFS: &v1.CephFSVolumeSource{ - Monitors: []string{"a", "b"}, - User: "user", - SecretRef: &v1.LocalObjectReference{ - Name: "name", - }, - SecretFile: "/etc/ceph/user.secret", - }, - }, - }, - }, - expectedName: "name", - expectedNs: "default", - expectedError: nil, - }, - } - for _, testcase := range tests { - resultName, resultNs, err := getSecretNameAndNamespace(testcase.spec, testcase.defaultNs) - if err != testcase.expectedError || resultName != testcase.expectedName || resultNs != testcase.expectedNs { - t.Errorf("%s failed: expected err=%v ns=%q name=%q, got %v/%q/%q", testcase.name, testcase.expectedError, testcase.expectedNs, testcase.expectedName, - err, resultNs, resultName) - } - } -} - -func TestGetAccessModes(t *testing.T) { - tmpDir, err := utiltesting.MkTmpdir("cephfs_test") - if err != nil { - t.Fatalf("error creating temp dir: %v", err) - } - defer os.RemoveAll(tmpDir) - - plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(t, tmpDir, nil, nil)) - - plug, err := plugMgr.FindPersistentPluginByName("kubernetes.io/cephfs") - if err != nil { - t.Fatal("Can't find the plugin by name") - } - modes := plug.GetAccessModes() - for _, v := range modes { - if !volumetest.ContainsAccessMode(modes, v) { - t.Errorf("Expected AccessModeTypes: %s", v) - } - } -} diff --git a/pkg/volume/cephfs/doc.go b/pkg/volume/cephfs/doc.go deleted file mode 100644 index f5dec711d28c..000000000000 --- a/pkg/volume/cephfs/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package cephfs contains the internal representation of Ceph file system -// (CephFS) volumes. -package cephfs // import "k8s.io/kubernetes/pkg/volume/cephfs" diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go index 2a62c9859b24..58ec455c67b3 100644 --- a/test/e2e/storage/drivers/in_tree.go +++ b/test/e2e/storage/drivers/in_tree.go @@ -359,162 +359,10 @@ func newISCSIServer(ctx context.Context, cs clientset.Interface, namespace strin return config, pod, ip, iqn } -// newRBDServer is a CephRBD-specific wrapper for CreateStorageServer. -func newRBDServer(ctx context.Context, cs clientset.Interface, namespace string) (config e2evolume.TestConfig, pod *v1.Pod, secret *v1.Secret, ip string) { - config = e2evolume.TestConfig{ - Namespace: namespace, - Prefix: "rbd", - ServerImage: imageutils.GetE2EImage(imageutils.VolumeRBDServer), - ServerPorts: []int{6789}, - ServerVolumes: map[string]string{ - "/lib/modules": "/lib/modules", - }, - ServerReadyMessage: "Ceph is ready", - } - pod, ip = e2evolume.CreateStorageServer(ctx, cs, config) - // create secrets for the server - secret = &v1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: config.Prefix + "-secret", - }, - Data: map[string][]byte{ - // from test/images/volumes-tester/rbd/keyring - "key": []byte("AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA=="), - }, - Type: "kubernetes.io/rbd", - } - - secret, err := cs.CoreV1().Secrets(config.Namespace).Create(ctx, secret, metav1.CreateOptions{}) - if err != nil { - framework.Failf("Failed to create secrets for Ceph RBD: %v", err) - } - - return config, pod, secret, ip -} - func (v *iSCSIVolume) DeleteVolume(ctx context.Context) { cleanUpVolumeServer(ctx, v.f, v.serverPod) } -// Ceph -type cephFSDriver struct { - driverInfo storageframework.DriverInfo -} - -type cephVolume struct { - serverPod *v1.Pod - serverIP string - secret *v1.Secret - f *framework.Framework -} - -var _ storageframework.TestDriver = &cephFSDriver{} -var _ storageframework.PreprovisionedVolumeTestDriver = &cephFSDriver{} -var _ storageframework.InlineVolumeTestDriver = &cephFSDriver{} -var _ storageframework.PreprovisionedPVTestDriver = &cephFSDriver{} - -// InitCephFSDriver returns cephFSDriver that implements TestDriver interface -func InitCephFSDriver() storageframework.TestDriver { - return &cephFSDriver{ - driverInfo: storageframework.DriverInfo{ - Name: "ceph", - InTreePluginName: "kubernetes.io/cephfs", - TestTags: []interface{}{feature.Volumes, framework.WithSerial()}, - MaxFileSize: storageframework.FileSizeMedium, - SupportedSizeRange: e2evolume.SizeRange{ - Min: "1Gi", - }, - SupportedFsType: sets.NewString( - "", // Default fsType - ), - Capabilities: map[storageframework.Capability]bool{ - storageframework.CapPersistence: true, - storageframework.CapExec: true, - storageframework.CapRWX: true, - storageframework.CapMultiPODs: true, - storageframework.CapMultiplePVsSameID: true, - }, - }, - } -} - -func (c *cephFSDriver) GetDriverInfo() *storageframework.DriverInfo { - return &c.driverInfo -} - -func (c *cephFSDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) { -} - -func (c *cephFSDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) *v1.VolumeSource { - cv, ok := e2evolume.(*cephVolume) - if !ok { - framework.Failf("Failed to cast test volume of type %T to the Ceph test volume", e2evolume) - } - - return &v1.VolumeSource{ - CephFS: &v1.CephFSVolumeSource{ - Monitors: []string{cv.serverIP + ":6789"}, - User: "kube", - SecretRef: &v1.LocalObjectReference{ - Name: cv.secret.Name, - }, - ReadOnly: readOnly, - }, - } -} - -func (c *cephFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) { - cv, ok := e2evolume.(*cephVolume) - if !ok { - framework.Failf("Failed to cast test volume of type %T to the Ceph test volume", e2evolume) - } - - ns := cv.f.Namespace - - return &v1.PersistentVolumeSource{ - CephFS: &v1.CephFSPersistentVolumeSource{ - Monitors: []string{cv.serverIP + ":6789"}, - User: "kube", - SecretRef: &v1.SecretReference{ - Name: cv.secret.Name, - Namespace: ns.Name, - }, - ReadOnly: readOnly, - }, - }, nil -} - -func (c *cephFSDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig { - return &storageframework.PerTestConfig{ - Driver: c, - Prefix: "cephfs", - Framework: f, - } -} - -func (c *cephFSDriver) CreateVolume(ctx context.Context, config *storageframework.PerTestConfig, volType storageframework.TestVolType) storageframework.TestVolume { - f := config.Framework - cs := f.ClientSet - ns := f.Namespace - - cfg, serverPod, secret, serverIP := newRBDServer(ctx, cs, ns.Name) - config.ServerConfig = &cfg - return &cephVolume{ - serverPod: serverPod, - serverIP: serverIP, - secret: secret, - f: f, - } -} - -func (v *cephVolume) DeleteVolume(ctx context.Context) { - cleanUpVolumeServerWithSecret(ctx, v.f, v.serverPod, v.secret) -} - // Hostpath type hostPathDriver struct { driverInfo storageframework.DriverInfo diff --git a/test/e2e/storage/in_tree_volumes.go b/test/e2e/storage/in_tree_volumes.go index 8f3502a7ac73..8534669ab47d 100644 --- a/test/e2e/storage/in_tree_volumes.go +++ b/test/e2e/storage/in_tree_volumes.go @@ -30,7 +30,6 @@ import ( var testDrivers = []func() storageframework.TestDriver{ drivers.InitNFSDriver, drivers.InitISCSIDriver, - drivers.InitCephFSDriver, drivers.InitHostPathDriver, drivers.InitHostPathSymlinkDriver, drivers.InitEmptydirDriver, diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index 989b49878a71..fcf34867ee48 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -222,8 +222,6 @@ const ( VolumeNFSServer // VolumeISCSIServer image VolumeISCSIServer - // VolumeRBDServer image - VolumeRBDServer ) func initImageConfigs(list RegistryList) (map[ImageID]Config, map[ImageID]Config) { @@ -265,7 +263,6 @@ func initImageConfigs(list RegistryList) (map[ImageID]Config, map[ImageID]Config configs[SdDummyExporter] = Config{list.GcRegistry, "sd-dummy-exporter", "v0.2.0"} configs[VolumeNFSServer] = Config{list.PromoterE2eRegistry, "volume/nfs", "1.4"} configs[VolumeISCSIServer] = Config{list.PromoterE2eRegistry, "volume/iscsi", "2.6"} - configs[VolumeRBDServer] = Config{list.PromoterE2eRegistry, "volume/rbd", "1.0.6"} // This adds more config entries. Those have no pre-defined ImageID number, // but will be used via ReplaceRegistryInImageURL when deploying