From d6e1faaeb21c3bf49495770e944c63d65b723b1e Mon Sep 17 00:00:00 2001 From: Bala FA Date: Tue, 27 Sep 2022 12:23:17 +0530 Subject: [PATCH] Add volume commands (#645) add volume commands Signed-off-by: Bala.FA Signed-off-by: Bala.FA --- cmd/kubectl-directpv/main.go | 11 +- cmd/kubectl-directpv/utils.go | 151 +++++++++++++++ cmd/kubectl-directpv/volumes.go | 151 +++++++++++++++ cmd/kubectl-directpv/volumes_list.go | 206 +++++++++++++++++++++ cmd/kubectl-directpv/volumes_purge.go | 130 +++++++++++++ cmd/kubectl-directpv/volumes_purge_test.go | 99 ++++++++++ go.mod | 3 +- go.sum | 8 - pkg/drive/drive.go | 3 +- pkg/k8s/k8s.go | 20 ++ pkg/types/aliases.go | 1 - pkg/utils/utils.go | 5 + 12 files changed, 774 insertions(+), 14 deletions(-) create mode 100644 cmd/kubectl-directpv/volumes.go create mode 100644 cmd/kubectl-directpv/volumes_list.go create mode 100644 cmd/kubectl-directpv/volumes_purge.go create mode 100644 cmd/kubectl-directpv/volumes_purge_test.go diff --git a/cmd/kubectl-directpv/main.go b/cmd/kubectl-directpv/main.go index a084a9cad..3ee95baaa 100644 --- a/cmd/kubectl-directpv/main.go +++ b/cmd/kubectl-directpv/main.go @@ -50,12 +50,17 @@ var ( jsonOutput = false yamlOutput = false noHeaders = false + allFlag = false ) var ( - drives, nodes, driveGlobs, nodeGlobs []string - driveSelectorValues, nodeSelectorValues []types.LabelValue - printer func(interface{}) error + driveArgs []string + nodeArgs []string + + driveSelectors []types.LabelValue + nodeSelectors []types.LabelValue + + printer func(interface{}) error ) var mainCmd = &cobra.Command{ diff --git a/cmd/kubectl-directpv/utils.go b/cmd/kubectl-directpv/utils.go index 0424e88bc..6af61a170 100644 --- a/cmd/kubectl-directpv/utils.go +++ b/cmd/kubectl-directpv/utils.go @@ -17,6 +17,7 @@ package main import ( + "context" "encoding/json" "fmt" "os" @@ -24,10 +25,19 @@ import ( "strings" "time" + "github.com/dustin/go-humanize" + directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" + "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/consts" + "github.com/minio/directpv/pkg/k8s" + "github.com/minio/directpv/pkg/types" "github.com/minio/directpv/pkg/utils" + "github.com/minio/directpv/pkg/volume" "github.com/mitchellh/go-homedir" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" ) const dot = "•" @@ -138,3 +148,144 @@ func openAuditFile(auditFile string) (*utils.SafeFile, error) { } return utils.NewSafeFile(path.Join(defaultAuditDir, fmt.Sprintf("%v.%v", auditFile, time.Now().UnixNano()))) } + +func printableString(s string) string { + if s == "" { + return "-" + } + return s +} + +func printableBytes(value int64) string { + if value == 0 { + return "-" + } + + return humanize.IBytes(uint64(value)) +} + +func getLabelValue(obj metav1.Object, key string) string { + if labels := obj.GetLabels(); labels != nil { + return labels[key] + } + return "" +} + +func matchVolumeStatus(volume types.Volume, statusList []string) bool { + return k8s.MatchTrueConditions( + volume.Status.Conditions, + []string{string(directpvtypes.VolumeConditionTypePublished), string(directpvtypes.VolumeConditionTypeStaged)}, + statusList, + ) +} + +func getFilteredVolumeList(ctx context.Context, filterFunc func(types.Volume) bool) ([]types.Volume, error) { + ctx, cancelFunc := context.WithCancel(ctx) + defer cancelFunc() + + resultCh, err := volume.ListVolumes( + ctx, + nodeSelectors, + driveSelectors, + podNameSelectors, + podNSSelectors, + k8s.MaxThreadCount, + ) + if err != nil { + return nil, err + } + + filteredVolumes := []types.Volume{} + for result := range resultCh { + if result.Err != nil { + return nil, result.Err + } + if matchVolumeStatus(result.Volume, volumeStatusSelectors) && filterFunc(result.Volume) { + filteredVolumes = append(filteredVolumes, result.Volume) + } + } + + return filteredVolumes, nil +} + +func getVolumesByNames(ctx context.Context, names []string) <-chan volume.ListVolumeResult { + resultCh := make(chan volume.ListVolumeResult) + go func() { + defer close(resultCh) + for _, name := range names { + volumeName := strings.TrimSpace(name) + vol, err := client.VolumeClient().Get(ctx, volumeName, metav1.GetOptions{}) + switch { + case err == nil: + resultCh <- volume.ListVolumeResult{Volume: *vol} + case apierrors.IsNotFound(err): + klog.V(5).Infof("No volume found by name %v", volumeName) + default: + klog.ErrorS(err, "unable to get volume", "volumeName", volumeName) + return + } + } + }() + return resultCh +} + +func processFilteredVolumes( + ctx context.Context, + names []string, + matchFunc func(*types.Volume) bool, + applyFunc func(*types.Volume) error, + processFunc func(context.Context, *types.Volume) error, + auditFile string, +) error { + var resultCh <-chan volume.ListVolumeResult + var err error + + if applyFunc == nil || processFunc == nil { + klog.Fatalf("Either applyFunc or processFunc must be provided. This should not happen.") + } + + ctx, cancelFunc := context.WithCancel(ctx) + defer cancelFunc() + + if len(names) == 0 { + resultCh, err = volume.ListVolumes(ctx, + nodeSelectors, + driveSelectors, + podNameSelectors, + podNSSelectors, + k8s.MaxThreadCount) + if err != nil { + return err + } + } else { + resultCh = getVolumesByNames(ctx, names) + } + + file, err := openAuditFile(auditFile) + if err != nil { + klog.ErrorS(err, "unable to open audit file", "auditFile", auditFile) + } + + defer func() { + if file != nil { + if err := file.Close(); err != nil { + klog.ErrorS(err, "unable to close audit file") + } + } + }() + + return volume.ProcessVolumes( + ctx, + resultCh, + func(volume *types.Volume) bool { + if matchVolumeStatus(*volume, volumeStatusSelectors) { + return matchFunc == nil || matchFunc(volume) + } + return false + }, + applyFunc, + processFunc, + file, + dryRun, + ) +} diff --git a/cmd/kubectl-directpv/volumes.go b/cmd/kubectl-directpv/volumes.go new file mode 100644 index 000000000..9561af7c9 --- /dev/null +++ b/cmd/kubectl-directpv/volumes.go @@ -0,0 +1,151 @@ +// This file is part of MinIO DirectPV +// Copyright (c) 2021, 2022 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + "errors" + "fmt" + "regexp" + "strings" + + directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" + "github.com/minio/directpv/pkg/ellipsis" + "github.com/minio/directpv/pkg/types" + "github.com/minio/directpv/pkg/utils" + "github.com/spf13/cobra" +) + +var ( + volumeStatusArgs []string + podNameArgs []string + podNSArgs []string + + volumeStatusSelectors []string + podNameSelectors []types.LabelValue + podNSSelectors []types.LabelValue +) + +var volumesCmd = &cobra.Command{ + Use: "volumes", + Short: "Manage DirectPV Volumes", + Aliases: []string{ + "volume", + "vol", + }, +} + +func init() { + volumesCmd.AddCommand(listVolumesCmd) + volumesCmd.AddCommand(purgeVolumesCmd) +} + +var ( + globRegexp = regexp.MustCompile(`(^|[^\\])[\*\?\[]`) + errGlobPatternUnsupported = errors.New("glob patterns are unsupported") +) + +func getSelectorValues(selectors []string) (values []types.LabelValue, err error) { + for _, selector := range selectors { + if globRegexp.MatchString(selector) { + return nil, errGlobPatternUnsupported + } + + result, err := ellipsis.Expand(selector) + if err != nil { + return nil, err + } + + for _, value := range result { + values = append(values, types.NewLabelValue(value)) + } + } + + return values, nil +} + +func getDriveSelectors() ([]types.LabelValue, error) { + var values []string + for i := range driveArgs { + if utils.TrimDevPrefix(driveArgs[i]) == "" { + return nil, fmt.Errorf("empty device name %v", driveArgs[i]) + } + values = append(values, utils.TrimDevPrefix(driveArgs[i])) + } + return getSelectorValues(values) +} + +func getNodeSelectors() ([]types.LabelValue, error) { + for i := range nodeArgs { + if utils.TrimDevPrefix(nodeArgs[i]) == "" { + return nil, fmt.Errorf("empty node name %v", nodeArgs[i]) + } + } + return getSelectorValues(nodeArgs) +} + +func getPodNameSelectors() ([]types.LabelValue, error) { + for i := range podNameArgs { + if utils.TrimDevPrefix(podNameArgs[i]) == "" { + return nil, fmt.Errorf("empty pod name %v", podNameArgs[i]) + } + } + return getSelectorValues(podNameArgs) +} + +func getPodNamespaceSelectors() ([]types.LabelValue, error) { + for i := range podNSArgs { + if utils.TrimDevPrefix(podNSArgs[i]) == "" { + return nil, fmt.Errorf("empty pod namespace %v", podNSArgs[i]) + } + } + return getSelectorValues(podNSArgs) +} + +func getVolumeStatusSelectors() ([]string, error) { + for _, status := range volumeStatusArgs { + switch directpvtypes.VolumeConditionType(strings.Title(status)) { + case directpvtypes.VolumeConditionTypePublished: + case directpvtypes.VolumeConditionTypeStaged: + case directpvtypes.VolumeConditionTypeReady: + default: + return nil, fmt.Errorf("unknown volume condition type %v", status) + } + } + return volumeStatusArgs, nil +} + +func validateVolumeSelectors() (err error) { + if driveSelectors, err = getDriveSelectors(); err != nil { + return err + } + + if nodeSelectors, err = getNodeSelectors(); err != nil { + return err + } + + if volumeStatusSelectors, err = getVolumeStatusSelectors(); err != nil { + return err + } + + if podNameSelectors, err = getPodNameSelectors(); err != nil { + return err + } + + podNSSelectors, err = getPodNamespaceSelectors() + + return err +} diff --git a/cmd/kubectl-directpv/volumes_list.go b/cmd/kubectl-directpv/volumes_list.go new file mode 100644 index 000000000..af9d9215a --- /dev/null +++ b/cmd/kubectl-directpv/volumes_list.go @@ -0,0 +1,206 @@ +// This file is part of MinIO DirectPV +// Copyright (c) 2021, 2022 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/fatih/color" + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jedib0t/go-pretty/v6/text" + directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" + "github.com/minio/directpv/pkg/consts" + "github.com/minio/directpv/pkg/k8s" + "github.com/minio/directpv/pkg/types" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" +) + +var ( + lostOnly bool + showPVC bool +) + +var listVolumesCmd = &cobra.Command{ + Use: "list", + Short: "List volumes served by " + consts.AppPrettyName + ".", + Example: strings.ReplaceAll( + `# List all staged and published volumes +$ kubectl {PLUGIN_NAME} volumes ls --status=staged,published + +# List all volumes from a particular node +$ kubectl {PLUGIN_NAME} volumes ls --node=node1 + +# Combine multiple filters using csv +$ kubectl {PLUGIN_NAME} vol ls --node=node1,node2 --status=staged --drive=/dev/nvme0n1 + +# List all published volumes by pod name +$ kubectl {PLUGIN_NAME} volumes ls --status=published --pod-name=minio-{1...3} + +# List all published volumes by pod namespace +$ kubectl {PLUGIN_NAME} volumes ls --status=published --pod-namespace=tenant-{1...3} + +# List all volumes provisioned based on drive and volume ellipses +$ kubectl {PLUGIN_NAME} volumes ls --drive /dev/xvd{a...d} --node node{1...4} + +# List all volumes and its PVC names +$ kubectl {PLUGIN_NAME} volumes ls --all --pvc + +# List all the "lost" volumes +$ kubectl {PLUGIN_NAME} volumes ls --lost + +# List all the "lost" volumes with their PVC names +$ kubectl {PLUGIN_NAME} volumes ls --lost --pvc`, + `{PLUGIN_NAME}`, + consts.AppName, + ), + RunE: func(c *cobra.Command, args []string) error { + if err := validateVolumeSelectors(); err != nil { + return err + } + return listVolumes(c.Context(), args) + }, + Aliases: []string{ + "ls", + }, +} + +func init() { + listVolumesCmd.PersistentFlags().StringSliceVarP(&driveArgs, "drive", "d", driveArgs, "Filter output by drives optionally in ellipses pattern.") + listVolumesCmd.PersistentFlags().StringSliceVarP(&nodeArgs, "node", "n", nodeArgs, "Filter output by nodes optionally in ellipses pattern.") + listVolumesCmd.PersistentFlags().StringSliceVarP(&volumeStatusArgs, "status", "s", volumeStatusArgs, fmt.Sprintf("Filter output by volume status. One of %v|%v|%v.", strings.ToLower(string(directpvtypes.VolumeConditionTypePublished)), strings.ToLower(string(directpvtypes.VolumeConditionTypeStaged)), strings.ToLower(string(directpvtypes.VolumeConditionTypeReady)))) + listVolumesCmd.PersistentFlags().StringSliceVarP(&podNameArgs, "pod-name", "", podNameArgs, "Filter output by pod names optionally in ellipses pattern.") + listVolumesCmd.PersistentFlags().StringSliceVarP(&podNSArgs, "pod-namespace", "", podNSArgs, "Filter output by pod namespaces optionally in ellipses pattern.") + listVolumesCmd.PersistentFlags().BoolVarP(&lostOnly, "lost", "", lostOnly, "Show lost volumes only.") + listVolumesCmd.PersistentFlags().BoolVarP(&showPVC, "pvc", "", showPVC, "Show PVC names of the corresponding volumes.") + listVolumesCmd.PersistentFlags().BoolVarP(&allFlag, "all", "a", allFlag, "List all volumes including not provisioned.") +} + +func getPVCName(ctx context.Context, volume types.Volume) string { + pv, err := k8s.KubeClient().CoreV1().PersistentVolumes().Get(ctx, volume.Name, metav1.GetOptions{}) + if err == nil && pv != nil && pv.Spec.ClaimRef != nil { + return pv.Spec.ClaimRef.Name + } + return "-" +} + +func listVolumes(ctx context.Context, args []string) error { + volumeList, err := getFilteredVolumeList( + ctx, + func(volume types.Volume) bool { + if lostOnly { + return k8s.IsCondition(volume.Status.Conditions, + string(directpvtypes.VolumeConditionTypeReady), + metav1.ConditionFalse, + string(directpvtypes.VolumeConditionReasonDriveLost), + string(directpvtypes.VolumeConditionMessageDriveLost), + ) + } + return allFlag || k8s.IsConditionStatus(volume.Status.Conditions, string(directpvtypes.VolumeConditionTypeReady), metav1.ConditionTrue) + }, + ) + if err != nil { + return err + } + + wrappedVolumeList := types.VolumeList{ + TypeMeta: metav1.TypeMeta{ + Kind: "List", + APIVersion: string(types.VersionLabelKey), + }, + Items: volumeList, + } + if yamlOutput || jsonOutput { + if err := printer(wrappedVolumeList); err != nil { + klog.ErrorS(err, "unable to marshal volumes", "format", outputFormat) + return err + } + return nil + } + + headers := table.Row{ + "VOLUME", + "CAPACITY", + "NODE", + "DRIVE", + "PODNAME", + "PODNAMESPACE", + "", + } + if wideOutput { + headers = append(headers, "DRIVENAME") + } + if showPVC { + headers = append(headers, "PVC") + } + + text.DisableColors() + t := table.NewWriter() + t.SetOutputMirror(os.Stdout) + if !noHeaders { + t.AppendHeader(headers) + } + + style := table.StyleColoredDark + style.Color.IndexColumn = text.Colors{text.FgHiBlue, text.BgHiBlack} + style.Color.Header = text.Colors{text.FgHiBlue, text.BgHiBlack} + t.SetStyle(style) + + for _, volume := range volumeList { + msg := "" + for _, c := range volume.Status.Conditions { + switch c.Type { + case string(directpvtypes.VolumeConditionTypeReady): + if c.Status != metav1.ConditionTrue { + if c.Message != "" { + msg = color.HiRedString("*" + c.Message) + } + } + } + } + row := []interface{}{ + volume.Name, // VOLUME + printableBytes(volume.Status.TotalCapacity), // CAPACITY + volume.Status.NodeName, // SERVER + getLabelValue(&volume, string(types.DrivePathLabelKey)), // DRIVE + printableString(volume.Labels[string(types.PodNameLabelKey)]), + printableString(volume.Labels[string(types.PodNSLabelKey)]), + msg, + } + if wideOutput { + row = append(row, getLabelValue(&volume, string(types.DriveLabelKey))) + } + if showPVC { + row = append(row, getPVCName(ctx, volume)) + } + t.AppendRow(row) + } + t.SortBy( + []table.SortBy{ + { + Name: "PODNAMESPACE", + Mode: table.Asc, + }, + }) + + t.Render() + return nil +} diff --git a/cmd/kubectl-directpv/volumes_purge.go b/cmd/kubectl-directpv/volumes_purge.go new file mode 100644 index 000000000..2e5940ef5 --- /dev/null +++ b/cmd/kubectl-directpv/volumes_purge.go @@ -0,0 +1,130 @@ +// This file is part of MinIO DirectPV +// Copyright (c) 2021, 2022 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + "context" + "fmt" + "strings" + + "github.com/minio/directpv/pkg/client" + "github.com/minio/directpv/pkg/consts" + "github.com/minio/directpv/pkg/k8s" + "github.com/minio/directpv/pkg/types" + "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" +) + +var purgeVolumesCmd = &cobra.Command{ + Use: "purge", + Short: "Purge released and failed " + consts.AppName + "volumes. CAUTION: THIS MAY LEAD TO DATA LOSS", + Example: strings.ReplaceAll( + `# Purge all released|failed volumes +$ kubectl {PLUGIN_NAME} volumes purge --all + +# Purge the volume by its name(id) +$ kubectl {PLUGIN_NAME} volumes purge + +# Purge all released|failed volumes from a particular node +$ kubectl {PLUGIN_NAME} volumes purge --nodes=node1 + +# Combine multiple filters using csv +$ kubectl {PLUGIN_NAME} volumes purge --nodes=node1,node2 --drives=/dev/nvme0n1 + +# Purge all released|failed volumes by pod name +$ kubectl {PLUGIN_NAME} volumes purge --pod-name=minio-{1...3} + +# Purge all released|failed volumes by pod namespace +$ kubectl {PLUGIN_NAME} volumes purge --pod-namespace=tenant-{1...3} + +# Purge all released|failed volumes based on drive and volume ellipses +$ kubectl {PLUGIN_NAME} volumes purge --drives /dev/xvd{a...d} --nodes node-{1...4}`, + `{PLUGIN_NAME}`, + consts.AppName, + ), + RunE: func(c *cobra.Command, args []string) error { + if !allFlag && len(driveArgs) == 0 && len(nodeArgs) == 0 && len(podNameArgs) == 0 && len(podNSArgs) == 0 && len(args) == 0 { + return fmt.Errorf("atleast one of '--all', '--drives', '--nodes', '--pod-name' or '--pod-namespace' must be specified") + } + if err := validateVolumeSelectors(); err != nil { + return err + } + + return purgeVolumes(c.Context(), args) + }, +} + +func init() { + purgeVolumesCmd.PersistentFlags().StringSliceVarP(&driveArgs, "drive", "d", driveArgs, "Filter output by drives optionally in ellipses pattern.") + purgeVolumesCmd.PersistentFlags().StringSliceVarP(&nodeArgs, "node", "n", nodeArgs, "Filter output by nodes optionally in ellipses pattern.") + purgeVolumesCmd.PersistentFlags().BoolVarP(&allFlag, "all", "a", allFlag, "Purge all released|failed volumes.") + purgeVolumesCmd.PersistentFlags().StringSliceVarP(&podNameArgs, "pod-name", "", podNameArgs, "Filter output by pod names optionally in ellipses pattern.") + purgeVolumesCmd.PersistentFlags().StringSliceVarP(&podNSArgs, "pod-namespace", "", podNSArgs, "Filter output by pod namespaces optionally in ellipses pattern.") +} + +func purgeVolumes(ctx context.Context, names []string) error { + return processFilteredVolumes( + ctx, + names, + func(volume *types.Volume) bool { + pv, err := k8s.KubeClient().CoreV1().PersistentVolumes().Get(ctx, volume.Name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return true + } + klog.ErrorS(err, "unable to get PV for volume", "volumeName", volume.Name) + return false + } + switch pv.Status.Phase { + case corev1.VolumeReleased, corev1.VolumeFailed: + return true + default: + if !quiet { + klog.Infof("Skipping volume %v as associated PV is in %v phase", volume.Name, pv.Status.Phase) + } + return false + } + }, + func(volume *types.Volume) error { + finalizers := volume.GetFinalizers() + updatedFinalizers := []string{} + for _, f := range finalizers { + if f == consts.VolumeFinalizerPVProtection { + continue + } + updatedFinalizers = append(updatedFinalizers, f) + } + volume.SetFinalizers(updatedFinalizers) + return nil + }, + func(ctx context.Context, volume *types.Volume) error { + if _, err := client.VolumeClient().Update(ctx, volume, metav1.UpdateOptions{ + TypeMeta: types.NewVolumeTypeMeta(), + }); err != nil { + return err + } + if err := client.VolumeClient().Delete(ctx, volume.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil + }, + "drive-purge", + ) +} diff --git a/cmd/kubectl-directpv/volumes_purge_test.go b/cmd/kubectl-directpv/volumes_purge_test.go new file mode 100644 index 000000000..cb25a2399 --- /dev/null +++ b/cmd/kubectl-directpv/volumes_purge_test.go @@ -0,0 +1,99 @@ +// This file is part of MinIO DirectPV +// Copyright (c) 2021, 2022 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + "context" + "testing" + + "github.com/minio/directpv/pkg/client" + clientsetfake "github.com/minio/directpv/pkg/clientset/fake" + "github.com/minio/directpv/pkg/consts" + "github.com/minio/directpv/pkg/k8s" + "github.com/minio/directpv/pkg/types" + "github.com/minio/directpv/pkg/volume" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubernetesfake "k8s.io/client-go/kubernetes/fake" +) + +func TestVolumesPurge(t *testing.T) { + createTestVolume := func(volumeName string) *types.Volume { + return &types.Volume{ + TypeMeta: types.NewVolumeTypeMeta(), + ObjectMeta: metav1.ObjectMeta{ + Name: volumeName, + Finalizers: []string{ + string(consts.VolumeFinalizerPurgeProtection), + string(consts.VolumeFinalizerPVProtection), + }, + }, + Status: types.VolumeStatus{}, + } + } + + createTestPV := func(pvName string, phase corev1.PersistentVolumePhase) *corev1.PersistentVolume { + return &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvName, + }, + Status: corev1.PersistentVolumeStatus{ + Phase: phase, + }, + } + } + + testVolumeObjects := []runtime.Object{ + createTestVolume("volume-1"), + createTestVolume("volume-2"), + createTestVolume("volume-3"), + createTestVolume("volume-4"), + } + + testPVObjects := []runtime.Object{ + createTestPV("volume-1", corev1.VolumeReleased), + createTestPV("volume-2", corev1.VolumeFailed), + createTestPV("volume-3", corev1.VolumeBound), + } + + if err := validateVolumeSelectors(); err != nil { + t.Fatalf("validateVolumeSelectors failed with %v", err) + } + + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + + clientset := types.NewExtFakeClientset(clientsetfake.NewSimpleClientset(testVolumeObjects...)) + client.SetVolumeInterface(clientset.DirectpvLatest().DirectPVVolumes()) + client.SetDriveInterface(clientset.DirectpvLatest().DirectPVDrives()) + k8s.SetKubeInterface(kubernetesfake.NewSimpleClientset(testPVObjects...)) + + if err := purgeVolumes(ctx, nil); err != nil { + t.Fatal(err) + } + + volumes, err := volume.GetVolumeList(ctx, nil, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + + // all volumes except bound volume should be removed + if len(volumes) != 1 { + t.Fatalf("volume count: expected: 1, got: %v", len(volumes)) + } +} diff --git a/go.mod b/go.mod index aafdbc5c3..76827d14e 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( k8s.io/apiextensions-apiserver v0.24.3 k8s.io/apimachinery v0.24.3 k8s.io/client-go v0.24.3 + k8s.io/klog v1.0.0 k8s.io/klog/v2 v2.60.1 k8s.io/kube-openapi v0.0.0-20220627174259-011e075b9cb8 sigs.k8s.io/yaml v1.2.0 @@ -81,7 +82,7 @@ require ( github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml v1.9.3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect diff --git a/go.sum b/go.sum index 2a0ac0f50..c64c8f204 100644 --- a/go.sum +++ b/go.sum @@ -134,8 +134,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= @@ -403,8 +401,6 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mb0/glob v0.0.0-20160210091149-1eb79d2de6c4 h1:NK3O7S5FRD/wj7ORQ5C3Mx1STpyEMuFe+/F0Lakd1Nk= -github.com/mb0/glob v0.0.0-20160210091149-1eb79d2de6c4/go.mod h1:FqD3ES5hx6zpzDainDaHgkTIqrPaI9uX4CVWqYZoQjY= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= @@ -460,8 +456,6 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -617,11 +611,9 @@ go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4 go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= diff --git a/pkg/drive/drive.go b/pkg/drive/drive.go index ca1d1e098..e8c4253b5 100644 --- a/pkg/drive/drive.go +++ b/pkg/drive/drive.go @@ -26,6 +26,7 @@ import ( "github.com/minio/directpv/pkg/consts" "github.com/minio/directpv/pkg/k8s" "github.com/minio/directpv/pkg/types" + "github.com/minio/directpv/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -37,7 +38,7 @@ func NewDrive(name string, status types.DriveStatus) *types.Drive { types.UpdateLabels(drive, map[types.LabelKey]types.LabelValue{ types.NodeLabelKey: types.NewLabelValue(status.NodeName), - types.PathLabelKey: types.NewLabelValue(strings.TrimPrefix(status.Path, "/dev/")), + types.PathLabelKey: types.NewLabelValue(utils.TrimDevPrefix(status.Path)), types.VersionLabelKey: types.NewLabelValue(consts.LatestAPIVersion), types.CreatedByLabelKey: consts.DriverName, types.AccessTierLabelKey: types.NewLabelValue(string(status.AccessTier)), diff --git a/pkg/k8s/k8s.go b/pkg/k8s/k8s.go index a48bf5ef7..85d0609d3 100644 --- a/pkg/k8s/k8s.go +++ b/pkg/k8s/k8s.go @@ -151,6 +151,26 @@ func UpdateCondition(conditions []metav1.Condition, ctype string, status metav1. } } +// MatchTrueConditions matches whether types and status list are in a true conditions or not. +func MatchTrueConditions(conditions []metav1.Condition, types, statusList []string) bool { + for i := range types { + types[i] = strings.ToLower(types[i]) + } + for i := range statusList { + statusList[i] = strings.ToLower(statusList[i]) + } + + statusMatches := 0 + for _, condition := range conditions { + ctype := strings.ToLower(condition.Type) + if condition.Status == metav1.ConditionTrue && utils.StringIn(types, ctype) && utils.StringIn(statusList, ctype) { + statusMatches++ + } + } + + return statusMatches == len(statusList) +} + // BoolToConditionStatus converts boolean value to condition status. func BoolToConditionStatus(val bool) metav1.ConditionStatus { if val { diff --git a/pkg/types/aliases.go b/pkg/types/aliases.go index 5b9436825..685b5656f 100644 --- a/pkg/types/aliases.go +++ b/pkg/types/aliases.go @@ -29,7 +29,6 @@ var Versions = []string{ } var LatestAddToScheme = directpv.AddToScheme - type Interface = typeddirectpv.DirectpvV1beta1Interface type Client = typeddirectpv.DirectpvV1beta1Client diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 1cf65144d..c2bcb988f 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -21,6 +21,7 @@ import ( "io" "os" "path/filepath" + "strings" "sigs.k8s.io/yaml" ) @@ -90,3 +91,7 @@ func (safeFile *SafeFile) Close() error { } return os.Rename(safeFile.tempFile.Name(), safeFile.filename) } + +func TrimDevPrefix(name string) string { + return strings.TrimPrefix(name, "/dev/") +}