From 6d0a799b5c45cd9c64129e258dbc0f6988845a00 Mon Sep 17 00:00:00 2001 From: "Bala.FA" Date: Thu, 25 May 2023 09:56:23 +0530 Subject: [PATCH] Add declarative installation Signed-off-by: Bala.FA --- cmd/kubectl-directpv/install.go | 53 +++++-- cmd/kubectl-directpv/list_drives.go | 2 +- cmd/kubectl-directpv/list_volumes.go | 2 +- cmd/kubectl-directpv/migrate.go | 32 +++- cmd/kubectl-directpv/utils.go | 20 ++- docs/tools/install.sh | 228 +++++++++++++++++++++++++++ docs/tools/migrate.sh | 82 ++++++++++ pkg/installer/args.go | 81 ++++++---- pkg/installer/crd.go | 63 ++++---- pkg/installer/csidriver.go | 38 ++--- pkg/installer/daemonset.go | 130 +++++++-------- pkg/installer/deployment.go | 43 +++-- pkg/installer/installer.go | 4 +- pkg/installer/installer_test.go | 4 +- pkg/installer/migrate.go | 97 +++++++++++- pkg/installer/namespace.go | 18 +-- pkg/installer/psp.go | 42 ++--- pkg/installer/rbac.go | 107 ++++--------- pkg/installer/storageclass.go | 46 ++---- pkg/installer/utils.go | 17 +- pkg/legacy/client/list.go | 7 +- pkg/utils/utils.go | 28 +--- pkg/utils/utils_test.go | 68 -------- 23 files changed, 765 insertions(+), 447 deletions(-) create mode 100755 docs/tools/install.sh create mode 100755 docs/tools/migrate.sh delete mode 100644 pkg/utils/utils_test.go diff --git a/cmd/kubectl-directpv/install.go b/cmd/kubectl-directpv/install.go index 9f68c2430..57933fefb 100644 --- a/cmd/kubectl-directpv/install.go +++ b/cmd/kubectl-directpv/install.go @@ -35,6 +35,7 @@ import ( "github.com/minio/directpv/pkg/volume" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/version" ) @@ -52,6 +53,7 @@ var ( k8sVersion = "1.27.0" kubeVersion *version.Version legacyFlag bool + declarativeFlag bool ) var installCmd = &cobra.Command{ @@ -122,6 +124,8 @@ func init() { addOutputFormatFlag(installCmd, "Generate installation manifest. One of: yaml|json") installCmd.PersistentFlags().StringVar(&k8sVersion, "kube-version", k8sVersion, "Select the kubernetes version for manifest generation") installCmd.PersistentFlags().BoolVar(&legacyFlag, "legacy", legacyFlag, "Enable legacy mode (Used with '-o')") + installCmd.PersistentFlags().BoolVar(&declarativeFlag, "declarative", declarativeFlag, "Output YAML for declarative installation") + installCmd.PersistentFlags().MarkHidden("declarative") } func validateNodeSelectorArgs() error { @@ -254,22 +258,26 @@ func getLegacyFlag(ctx context.Context) bool { func installMain(ctx context.Context) { legacyFlag = getLegacyFlag(ctx) - auditFile := fmt.Sprintf("install.log.%v", time.Now().UTC().Format(time.RFC3339Nano)) - file, err := openAuditFile(auditFile) - if err != nil { - utils.Eprintf(quietFlag, true, "unable to open audit file %v; %v\n", auditFile, err) - utils.Eprintf(false, false, "%v\n", color.HiYellowString("Skipping audit logging")) - } + var file *utils.SafeFile + var err error + if dryRunPrinter == nil && !declarativeFlag { + auditFile := fmt.Sprintf("install.log.%v", time.Now().UTC().Format(time.RFC3339Nano)) + file, err = openAuditFile(auditFile) + if err != nil { + utils.Eprintf(quietFlag, true, "unable to open audit file %v; %v\n", auditFile, err) + utils.Eprintf(false, false, "%v\n", color.HiYellowString("Skipping audit logging")) + } - defer func() { - if file != nil { - if err := file.Close(); err != nil { - utils.Eprintf(quietFlag, true, "unable to close audit file; %v\n", err) + defer func() { + if file != nil { + if err := file.Close(); err != nil { + utils.Eprintf(quietFlag, true, "unable to close audit file; %v\n", err) + } } - } - }() + }() + } - args, err := installer.NewArgs(image, file) + args := installer.NewArgs(image) if err != nil { utils.Eprintf(quietFlag, true, "%v\n", err) os.Exit(1) @@ -285,12 +293,27 @@ func installMain(ctx context.Context) { args.Quiet = quietFlag args.KubeVersion = kubeVersion args.Legacy = legacyFlag - args.DryRunPrinter = dryRunPrinter + if file != nil { + args.ObjectWriter = file + } + if dryRunPrinter != nil { + args.DryRun = true + if outputFormat == "yaml" { + args.ObjectConverter = func(obj runtime.Object) ([]byte, error) { + return utils.ToYAML(obj) + } + } else { + args.ObjectConverter = func(obj runtime.Object) ([]byte, error) { + return utils.ToJSON(obj) + } + } + } + args.Declarative = declarativeFlag var failed bool var installedComponents []installer.Component var wg sync.WaitGroup - if dryRunPrinter == nil && !quietFlag { + if dryRunPrinter == nil && !declarativeFlag && !quietFlag { m := newProgressModel(true) teaProgram := tea.NewProgram(m) wg.Add(1) diff --git a/cmd/kubectl-directpv/list_drives.go b/cmd/kubectl-directpv/list_drives.go index 8705c660a..4b08d0455 100644 --- a/cmd/kubectl-directpv/list_drives.go +++ b/cmd/kubectl-directpv/list_drives.go @@ -138,7 +138,7 @@ func listDrivesMain(ctx context.Context) { driveList := types.DriveList{ TypeMeta: metav1.TypeMeta{ Kind: "List", - APIVersion: string(directpvtypes.VersionLabelKey), + APIVersion: "v1", }, Items: drives, } diff --git a/cmd/kubectl-directpv/list_volumes.go b/cmd/kubectl-directpv/list_volumes.go index d5f1927c1..88867c459 100644 --- a/cmd/kubectl-directpv/list_volumes.go +++ b/cmd/kubectl-directpv/list_volumes.go @@ -179,7 +179,7 @@ func listVolumesMain(ctx context.Context) { volumeList := types.VolumeList{ TypeMeta: metav1.TypeMeta{ Kind: "List", - APIVersion: string(directpvtypes.VersionLabelKey), + APIVersion: "v1", }, Items: volumes, } diff --git a/cmd/kubectl-directpv/migrate.go b/cmd/kubectl-directpv/migrate.go index f69fa89cc..5c0208811 100644 --- a/cmd/kubectl-directpv/migrate.go +++ b/cmd/kubectl-directpv/migrate.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "strings" + "time" "github.com/minio/directpv/pkg/consts" "github.com/minio/directpv/pkg/installer" @@ -28,6 +29,8 @@ import ( "github.com/spf13/cobra" ) +var retainFlag bool + var migrateCmd = &cobra.Command{ Use: "migrate", Short: "Migrate drives and volumes from legacy DirectCSI", @@ -51,13 +54,14 @@ func init() { migrateCmd.PersistentFlags().SortFlags = false addDryRunFlag(migrateCmd, "Run in dry run mode") + migrateCmd.PersistentFlags().BoolVar(&retainFlag, "retain", retainFlag, "retain legacy CRD after migration") } func migrateMain(ctx context.Context) { if err := installer.Migrate(ctx, &installer.Args{ Quiet: quietFlag, Legacy: true, - }); err != nil { + }, false); err != nil { utils.Eprintf(quietFlag, true, "migration failed; %v", err) os.Exit(1) } @@ -65,4 +69,30 @@ func migrateMain(ctx context.Context) { if !quietFlag { fmt.Println("Migration successful; Please restart the pods in '" + consts.AppName + "' namespace.") } + + if retainFlag { + return + } + + suffix := time.Now().Format(time.RFC3339) + + drivesBackupFile := "directcsidrives-" + suffix + ".yaml" + backupCreated, err := installer.RemoveLegacyDrives(ctx, drivesBackupFile) + if err != nil { + utils.Eprintf(quietFlag, true, "unable to remove legacy drive CRDs; %v", err) + os.Exit(1) + } + if backupCreated && !quietFlag { + fmt.Println("Legacy drive CRDs backed up to", drivesBackupFile) + } + + volumesBackupFile := "directcsivolumes-" + suffix + ".yaml" + backupCreated, err = installer.RemoveLegacyVolumes(ctx, volumesBackupFile) + if err != nil { + utils.Eprintf(quietFlag, true, "unable to remove legacy volume CRDs; %v", err) + os.Exit(1) + } + if backupCreated && !quietFlag { + fmt.Println("Legacy volume CRDs backed up to", volumesBackupFile) + } } diff --git a/cmd/kubectl-directpv/utils.go b/cmd/kubectl-directpv/utils.go index 79543d8bf..43caec3c5 100644 --- a/cmd/kubectl-directpv/utils.go +++ b/cmd/kubectl-directpv/utils.go @@ -28,16 +28,27 @@ import ( "github.com/minio/directpv/pkg/consts" "github.com/minio/directpv/pkg/utils" "github.com/mitchellh/go-homedir" + "k8s.io/klog/v2" ) const dot = "•" func printYAML(obj interface{}) { - fmt.Print(utils.MustGetYAML(obj)) + data, err := utils.ToYAML(obj) + if err != nil { + klog.Fatalf("unable to marshal object to YAML; %w", err) + } + + fmt.Print(string(data)) } func printJSON(obj interface{}) { - fmt.Print(utils.MustGetJSON(obj)) + data, err := utils.ToJSON(obj) + if err != nil { + klog.Fatalf("unable to marshal object to JSON; %w", err) + } + + fmt.Print(string(data)) } func getDefaultAuditDir() (string, error) { @@ -109,7 +120,10 @@ func validateOutputFormat(isWideSupported bool) error { case "json": dryRunPrinter = printJSON default: - return errors.New("--output flag value must be one of wide|json|yaml or empty") + if isWideSupported { + return errors.New("--output flag value must be one of wide|json|yaml or empty") + } + return errors.New("--output flag value must be one of yaml|json") } return nil } diff --git a/docs/tools/install.sh b/docs/tools/install.sh new file mode 100755 index 000000000..dce274117 --- /dev/null +++ b/docs/tools/install.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash +# This file is part of MinIO DirectPV +# Copyright (c) 2023 MinIO, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# +# This script installs or upgrades DirectPV declaratively. +# + +set -e + +declare -a plugin_cmd directpv_args +apply_flag=0 + +function run_plugin_cmd() { + "${plugin_cmd[@]}" "$@" +} + +function is_legacy_drive_found() { + if ! kubectl get --ignore-not-found=true crd directcsidrives.direct.csi.min.io --no-headers -o NAME | grep -q .; then + return 1 + fi + + exists=$(kubectl get directcsidrives --ignore-not-found=true -o go-template='{{range .items}}{{1}}{{break}}{{end}}') + [ -n "${exists}" ] +} + +function is_legacy_volume_found() { + if ! kubectl get --ignore-not-found=true crd directcsivolumes.direct.csi.min.io --no-headers -o NAME | grep -q .; then + return 1 + fi + + exists=$(kubectl get directcsivolumes --ignore-not-found=true -o go-template='{{range .items}}{{1}}{{break}}{{end}}') + [ -n "${exists}" ] +} + +function is_migrated_volume_found() { + if ! kubectl get --ignore-not-found=true crd directpvvolumes.directpv.min.io --no-headers -o NAME | grep -q .; then + return 1 + fi + + exists=$(kubectl get directpvvolumes.directpv.min.io --selector=directpv.min.io/migrated=true --ignore-not-found=true -o go-template='{{range .items}}{{1}}{{break}}{{end}}') + [ -n "${exists}" ] +} + +declare -A fsuuidDriveNameMap driveNameFSUUIDMap + +function check_directcsi_consistency() { + ecount=0 + # shellcheck disable=SC2207 + names=( $(kubectl get directcsidrives -o go-template='{{range .items}}{{if or (eq .status.driveStatus "Ready") (eq .status.driveStatus "InUse") }}{{.metadata.name}} {{end}}{{end}}') ) + for name in "${names[@]}"; do + fsuuid=$(kubectl get directcsidrives "${name}" -o go-template='{{.status.filesystemUUID}}') + + if [ -z "${fsuuid}" ]; then + echo "[ERROR] drive ${name}: empty fsuuid" + (( ++ecount )) + continue + fi + + if ! echo "${fsuuid}" | grep -q -E "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; then + echo "[ERROR] drive ${name}: invalid fsuuid ${fsuuid}" + (( ++ecount )) + continue + fi + + if [ -n "${fsuuidDriveNameMap[$fsuuid]}" ]; then + echo "[ERROR] drive ${name}: duplicate FSUUID ${fsuuid} found" + (( ++ecount )) + continue + fi + + fsuuidDriveNameMap[$fsuuid]=$name + driveNameFSUUIDMap[$name]=$fsuuid + done + + if [ "${ecount}" -gt 0 ]; then + exit 1 + fi + + for nameDrive in $(kubectl get directcsivolumes -o go-template='{{range .items}}{{.metadata.name}}={{.status.drive}} {{end}}'); do + name=$(echo "${nameDrive}" | cut -d= -f1) + drive=$(echo "${nameDrive}" | cut -d= -f2) + + if [ -z "${drive}" ]; then + echo "[ERROR] volume ${name}: empty drive name" + (( ++ecount )) + continue + fi + + if [ -z "${driveNameFSUUIDMap[$drive]}" ]; then + echo "[ERROR] volume ${name}: FSUUID not found for drive ${drive}" + (( ++ecount )) + continue + fi + done + + if [ "${ecount}" -gt 0 ]; then + exit 1 + fi +} + +function init() { + help_flag=0 + for arg in "$@"; do + if [ "${arg}" != "apply" ]; then + directpv_args+=("${arg}") + if [ "${arg}" == "--help" ]; then + help_flag=1 + fi + else + apply_flag=1 + fi + done + + if [ "${help_flag}" -eq 1 ]; then + cat << EOF +USAGE: + install.sh [INSTALL-FLAGS] [apply] + +ARGUMENTS: + INSTALL-FLAGS Optional DirectPV installation flags. + apply If present, apply manifest file after generation. + +EXAMPLES: + # Generate DirectPV manifests. + $ install.sh + + # Generate DirectPV manifests with private registry. + $ install.sh --registry private-registry.io --org org-name + + # Generate DirectPV manifests with node-selector. + $ install.sh --node-selector node-label-key=node-label-value + + # Generate and apply DirectPV manifests. + $ install.sh apply +EOF + exit 255 + fi + + if ! which kubectl >/dev/null 2>&1; then + echo "kubectl not found; please install" + exit 255 + fi + + if kubectl directpv --version >/dev/null 2>&1; then + plugin_cmd=( kubectl directpv ) + elif which kubectl-directpv >/dev/null 2>&1; then + plugin_cmd=( kubectl-directpv ) + elif ./kubectl-directpv --version >/dev/null 2>&1; then + plugin_cmd=( ./kubectl-directpv ) + else + echo "kubectl directpv plugin not found; please install" + exit 255 + fi +} + +function wait_for_crd() { + while ! kubectl get --ignore-not-found=true crd directpvdrives.directpv.min.io --no-headers -o NAME | grep -q .; do + echo " ...waiting for drive CRD to be created" + sleep 1 + done + + while ! kubectl get --ignore-not-found=true crd directpvvolumes.directpv.min.io --no-headers -o NAME | grep -q .; do + echo " ...waiting for volume CRD to be created" + sleep 1 + done +} + +function main() { + echo "* Probe legacy DirectCSI" + legacy=0 + migrate=0 + if is_legacy_drive_found || is_legacy_volume_found; then + legacy=1 + migrate=1 + check_directcsi_consistency + elif is_migrated_volume_found; then + legacy=1 + fi + + echo "* Generate DirectPV manifests to be saved at directpv.yaml" + if [[ "${legacy}" -eq 0 ]]; then + run_plugin_cmd install --declarative "${directpv_args[@]}" > directpv.yaml + else + run_plugin_cmd install --legacy --declarative "${directpv_args[@]}" > directpv.yaml + fi + + if [ "${apply_flag}" -eq 1 ]; then + echo "* To install/upgrade DirectPV, run 'kubectl apply -f directpv.yaml'" + if [[ "${migrate}" -eq 1 ]]; then + echo "* After install/upgrade, run 'migrate.sh' to complete migration" + fi + + return 0 + fi + + echo "* Apply manifests from directpv.yaml" + if ! kubectl apply -f directpv.yaml; then + return 1 + fi + + if [[ "${migrate}" -eq 0 ]]; then + return 0 + fi + + echo "* Migrate legacy drives and volumes" + wait_for_crd + if ! run_plugin_cmd migrate; then + return 1 + fi + kubectl -n directpv delete pods --all +} + +init "$@" +main "$@" diff --git a/docs/tools/migrate.sh b/docs/tools/migrate.sh new file mode 100755 index 000000000..fcfb2e33d --- /dev/null +++ b/docs/tools/migrate.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# This file is part of MinIO DirectPV +# Copyright (c) 2023 MinIO, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# +# This script migrates legacy drives and volumes. +# +# Usage: +# migrate.sh +# + +set -e + +declare -a plugin_cmd + +function run_plugin_cmd() { + "${plugin_cmd[@]}" "$@" +} + +function is_legacy_drive_found() { + if ! kubectl get --ignore-not-found=true crd directcsidrives.direct.csi.min.io --no-headers -o NAME | grep -q .; then + return 1 + fi + + exists=$(kubectl get directcsidrives --ignore-not-found=true -o go-template='{{range .items}}{{1}}{{break}}{{end}}') + [ -n "${exists}" ] +} + +function is_legacy_volume_found() { + if ! kubectl get --ignore-not-found=true crd directcsivolumes.direct.csi.min.io --no-headers -o NAME | grep -q .; then + return 1 + fi + + exists=$(kubectl get directcsivolumes --ignore-not-found=true -o go-template='{{range .items}}{{1}}{{break}}{{end}}') + [ -n "${exists}" ] +} + +function init() { + if ! which kubectl >/dev/null 2>&1; then + echo "kubectl not found; please install" + exit 255 + fi + + if kubectl directpv --version >/dev/null 2>&1; then + plugin_cmd=( kubectl directpv ) + elif which kubectl-directpv >/dev/null 2>&1; then + plugin_cmd=( kubectl-directpv ) + elif ./kubectl-directpv --version >/dev/null 2>&1; then + plugin_cmd=( ./kubectl-directpv ) + else + echo "kubectl directpv plugin not found; please install" + exit 255 + fi +} + +function main() { + if ! is_legacy_drive_found && ! is_legacy_volume_found; then + return 0 + fi + + if ! run_plugin_cmd migrate; then + return 1 + fi + + kubectl -n directpv delete pods --all +} + +init "$@" +main "$@" diff --git a/pkg/installer/args.go b/pkg/installer/args.go index acebe362e..838754c6a 100644 --- a/pkg/installer/args.go +++ b/pkg/installer/args.go @@ -18,10 +18,13 @@ package installer import ( "errors" + "fmt" "io" "path" + "github.com/minio/directpv/pkg/utils" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/version" ) @@ -34,40 +37,37 @@ const ( // Args represents DirectPV installation arguments. type Args struct { - image string - auditWriter io.Writer + image string // Optional arguments - Registry string - Org string - ImagePullSecrets []string - NodeSelector map[string]string - Tolerations []corev1.Toleration - SeccompProfile string - AppArmorProfile string - Quiet bool - KubeVersion *version.Version - Legacy bool + Registry string + Org string + ImagePullSecrets []string + NodeSelector map[string]string + Tolerations []corev1.Toleration + SeccompProfile string + AppArmorProfile string + Quiet bool + KubeVersion *version.Version + Legacy bool + ObjectWriter io.Writer + DryRun bool + Declarative bool + ObjectConverter func(runtime.Object) ([]byte, error) + ProgressCh chan<- Message + ForceUninstall bool + podSecurityAdmission bool csiProvisionerImage string nodeDriverRegistrarImage string livenessProbeImage string csiResizerImage string - ProgressCh chan<- Message - ForceUninstall bool - DryRunPrinter func(interface{}) -} - -func (args Args) dryRun() bool { - return args.DryRunPrinter != nil } // NewArgs creates arguments for DirectPV installation. -func NewArgs(image string, auditWriter io.Writer) (*Args, error) { - args := &Args{ - image: image, - auditWriter: auditWriter, - +func NewArgs(image string) *Args { + return &Args{ + image: image, Registry: "quay.io", Org: "minio", @@ -76,11 +76,6 @@ func NewArgs(image string, auditWriter io.Writer) (*Args, error) { livenessProbeImage: livenessProbeImage, csiResizerImage: csiResizerImage, } - - if err := args.validate(); err != nil { - return nil, err - } - return args, nil } func (args *Args) validate() error { @@ -88,13 +83,37 @@ func (args *Args) validate() error { return errors.New("image name must be provided") } - if args.auditWriter == nil { - return errors.New("audit writer must be provided") + if !args.DryRun && !args.Declarative && args.ObjectWriter == nil { + return errors.New("object writer must be provided") + } + + if args.DryRun && args.ObjectConverter == nil { + return errors.New("object converter must be provided") } return nil } +func (args *Args) writeObject(obj runtime.Object) (err error) { + var data []byte + if args.ObjectConverter != nil { + data, err = args.ObjectConverter(obj) + } else { + data, err = utils.ToYAML(obj) + } + if err != nil { + return err + } + + if args.ObjectWriter != nil { + _, err = args.ObjectWriter.Write(data) + } else { + fmt.Print(string(data)) + } + + return err +} + func (args *Args) getImagePullSecrets() (refs []corev1.LocalObjectReference) { for _, name := range args.ImagePullSecrets { refs = append(refs, corev1.LocalObjectReference{Name: name}) diff --git a/pkg/installer/crd.go b/pkg/installer/crd.go index eb10c5f3d..b68636d5c 100644 --- a/pkg/installer/crd.go +++ b/pkg/installer/crd.go @@ -20,7 +20,6 @@ import ( "context" _ "embed" "fmt" - "io" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" "github.com/minio/directpv/pkg/client" @@ -29,7 +28,6 @@ import ( "github.com/minio/directpv/pkg/initrequest" "github.com/minio/directpv/pkg/k8s" "github.com/minio/directpv/pkg/node" - "github.com/minio/directpv/pkg/utils" "github.com/minio/directpv/pkg/volume" "k8s.io/apiextensions-apiserver/pkg/apihelpers" apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -114,19 +112,18 @@ func getLatestCRDVersionObject( } func updateCRD( - ctx context.Context, existingCRD, newCRD *apiextensions.CustomResourceDefinition, -) (*apiextensions.CustomResourceDefinition, error) { +) (*apiextensions.CustomResourceDefinition, bool, error) { existingCRDStorageVersion, err := apihelpers.GetCRDStorageVersion(existingCRD) if err != nil { - return nil, err + return nil, false, err } setNoneConversionStrategy(existingCRD) // CRD is already in the latest version if existingCRDStorageVersion == consts.LatestAPIVersion { - return existingCRD, nil + return existingCRD, true, nil } var versionEntryFound bool @@ -143,12 +140,12 @@ func updateCRD( if !versionEntryFound { latestVersionObject, err := getLatestCRDVersionObject(newCRD) if err != nil { - return nil, err + return nil, false, err } existingCRD.Spec.Versions = append(existingCRD.Spec.Versions, latestVersionObject) } - return k8s.CRDClient().Update(ctx, existingCRD, metav1.UpdateOptions{}) + return existingCRD, false, nil } func createCRDs(ctx context.Context, args *Args) (err error) { @@ -162,16 +159,16 @@ func createCRDs(ctx context.Context, args *Args) (err error) { if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object, &crd); err != nil { return err } - - if args.dryRun() { - updateLabels( - &crd, - map[directpvtypes.LabelKey]directpvtypes.LabelValue{ - directpvtypes.VersionLabelKey: consts.LatestAPIVersion, - }, - ) - args.DryRunPrinter(crd) - return nil + setNoneConversionStrategy(&crd) + updateLabels( + &crd, + map[directpvtypes.LabelKey]directpvtypes.LabelValue{ + directpvtypes.VersionLabelKey: consts.LatestAPIVersion, + }, + ) + + if args.DryRun { + return args.writeObject(&crd) } existingCRD, err := k8s.CRDClient().Get(ctx, crd.Name, metav1.GetOptions{}) @@ -180,33 +177,45 @@ func createCRDs(ctx context.Context, args *Args) (err error) { return err } - setNoneConversionStrategy(&crd) if !sendProgressMessage(ctx, args.ProgressCh, fmt.Sprintf("Registering %s CRD", crd.Name), step, nil) { return errSendProgress } - _, err := k8s.CRDClient().Create(ctx, &crd, metav1.CreateOptions{}) - if err != nil { - return err + + if !args.Declarative { + _, err := k8s.CRDClient().Create(ctx, &crd, metav1.CreateOptions{}) + if err != nil { + return err + } } + if !sendProgressMessage(ctx, args.ProgressCh, fmt.Sprintf("Registered %s CRD", crd.Name), step, crdComponent(crd.Name)) { return errSendProgress } - _, err = io.WriteString(args.auditWriter, utils.MustGetYAML(crd)) - return err + return args.writeObject(&crd) } if !sendProgressMessage(ctx, args.ProgressCh, fmt.Sprintf("Updating %s CRD", crd.Name), step, nil) { return errSendProgress } - updatedCRD, err := updateCRD(ctx, existingCRD, &crd) + + updatedCRD, isLatest, err := updateCRD(existingCRD, &crd) if err != nil { return err } + + if !args.Declarative && !isLatest { + updatedCRD, err = k8s.CRDClient().Update(ctx, updatedCRD, metav1.UpdateOptions{}) + if err != nil { + return err + } + } + if !sendProgressMessage(ctx, args.ProgressCh, fmt.Sprintf("Updated %s CRD", crd.Name), step, crdComponent(crd.Name)) { return errSendProgress } - _, err = io.WriteString(args.auditWriter, utils.MustGetYAML(updatedCRD)) - return err + + updatedCRD.TypeMeta = crd.TypeMeta + return args.writeObject(updatedCRD) } if err := register(drivesYAML, 1); err != nil { diff --git a/pkg/installer/csidriver.go b/pkg/installer/csidriver.go index 6af5430f6..1966417dd 100644 --- a/pkg/installer/csidriver.go +++ b/pkg/installer/csidriver.go @@ -20,12 +20,10 @@ import ( "context" "errors" "fmt" - "io" "github.com/minio/directpv/pkg/consts" "github.com/minio/directpv/pkg/k8s" legacyclient "github.com/minio/directpv/pkg/legacy/client" - "github.com/minio/directpv/pkg/utils" storagev1 "k8s.io/api/storage/v1" storagev1beta1 "k8s.io/api/storage/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -107,21 +105,14 @@ func doCreateCSIDriver(ctx context.Context, args *Args, version string, legacy b }, } - if args.dryRun() { - args.DryRunPrinter(csiDriver) - return nil - } - - _, err := k8s.KubeClient().StorageV1().CSIDrivers().Create(ctx, csiDriver, metav1.CreateOptions{}) - if err != nil { - if apierrors.IsAlreadyExists(err) { - err = nil + if !args.DryRun && !args.Declarative { + _, err := k8s.KubeClient().StorageV1().CSIDrivers().Create(ctx, csiDriver, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err } - return err } - _, err = io.WriteString(args.auditWriter, utils.MustGetYAML(csiDriver)) - return err + return args.writeObject(csiDriver) case "v1beta1": csiDriver := &storagev1beta1.CSIDriver{ @@ -145,21 +136,14 @@ func doCreateCSIDriver(ctx context.Context, args *Args, version string, legacy b }, } - if args.dryRun() { - args.DryRunPrinter(csiDriver) - return nil - } - - _, err := k8s.KubeClient().StorageV1beta1().CSIDrivers().Create(ctx, csiDriver, metav1.CreateOptions{}) - if err != nil { - if apierrors.IsAlreadyExists(err) { - err = nil + if !args.DryRun && !args.Declarative { + _, err := k8s.KubeClient().StorageV1beta1().CSIDrivers().Create(ctx, csiDriver, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err } - return err } - _, err = io.WriteString(args.auditWriter, utils.MustGetYAML(csiDriver)) - return err + return args.writeObject(csiDriver) default: return errCSIDriverVersionUnsupported @@ -168,7 +152,7 @@ func doCreateCSIDriver(ctx context.Context, args *Args, version string, legacy b func createCSIDriver(ctx context.Context, args *Args) (err error) { version := "v1" - if args.dryRun() { + if args.DryRun { if args.KubeVersion.Major() >= 1 && args.KubeVersion.Minor() < 19 { version = "v1beta1" } diff --git a/pkg/installer/daemonset.go b/pkg/installer/daemonset.go index ca5f8674a..8e0677114 100644 --- a/pkg/installer/daemonset.go +++ b/pkg/installer/daemonset.go @@ -19,12 +19,10 @@ package installer import ( "context" "fmt" - "io" "github.com/minio/directpv/pkg/consts" "github.com/minio/directpv/pkg/k8s" legacyclient "github.com/minio/directpv/pkg/legacy/client" - "github.com/minio/directpv/pkg/utils" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -205,7 +203,7 @@ func livenessProbeContainer(image string) corev1.Container { } } -func newDaemonset(podSpec corev1.PodSpec, name, appArmorProfile string) *appsv1.DaemonSet { +func newDaemonset(podSpec corev1.PodSpec, name, selectorValue, appArmorProfile string) *appsv1.DaemonSet { annotations := map[string]string{createdByLabel: pluginName} if appArmorProfile != "" { // AppArmor profiles need to be specified per-container @@ -213,7 +211,6 @@ func newDaemonset(podSpec corev1.PodSpec, name, appArmorProfile string) *appsv1. annotations["container.apparmor.security.beta.kubernetes.io/"+container.Name] = "localhost/" + appArmorProfile } } - selectorValue := fmt.Sprintf("%v-%v", consts.Identity, getRandSuffix()) return &appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{ @@ -240,6 +237,9 @@ func newDaemonset(podSpec corev1.PodSpec, name, appArmorProfile string) *appsv1. }, Spec: podSpec, }, + UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + Type: appsv1.RollingUpdateDaemonSetStrategyType, + }, }, Status: appsv1.DaemonSetStatus{}, } @@ -280,25 +280,40 @@ func doCreateDaemonset(ctx context.Context, args *Args) (err error) { Tolerations: args.Tolerations, } - daemonset := newDaemonset(podSpec, consts.NodeServerName, args.AppArmorProfile) + var selectorValue string + if !args.DryRun { + daemonset, err := k8s.KubeClient().AppsV1().DaemonSets(namespace).Get( + ctx, consts.NodeServerName, metav1.GetOptions{}, + ) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err == nil { + if !args.Declarative { + return nil + } - if args.dryRun() { - args.DryRunPrinter(daemonset) - return nil + if daemonset.Spec.Selector != nil && daemonset.Spec.Selector.MatchLabels != nil { + selectorValue = daemonset.Spec.Selector.MatchLabels[selectorKey] + } + } + } + if selectorValue == "" { + selectorValue = fmt.Sprintf("%v-%v", consts.Identity, getRandSuffix()) } - _, err = k8s.KubeClient().AppsV1().DaemonSets(namespace).Create( - ctx, daemonset, metav1.CreateOptions{}, - ) - if err != nil { - if apierrors.IsAlreadyExists(err) { - err = nil + daemonset := newDaemonset(podSpec, consts.NodeServerName, selectorValue, args.AppArmorProfile) + + if !args.DryRun && !args.Declarative { + _, err = k8s.KubeClient().AppsV1().DaemonSets(namespace).Create( + ctx, daemonset, metav1.CreateOptions{}, + ) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err } - return err } - _, err = io.WriteString(args.auditWriter, utils.MustGetYAML(daemonset)) - return err + return args.writeObject(daemonset) } func doCreateLegacyDaemonset(ctx context.Context, args *Args) (err error) { @@ -328,78 +343,67 @@ func doCreateLegacyDaemonset(ctx context.Context, args *Args) (err error) { Tolerations: args.Tolerations, } - daemonset := newDaemonset(podSpec, consts.LegacyNodeServerName, args.AppArmorProfile) - - if args.dryRun() { - args.DryRunPrinter(daemonset) - return nil - } + var selectorValue string + if !args.DryRun { + daemonset, err := k8s.KubeClient().AppsV1().DaemonSets(namespace).Get( + ctx, consts.LegacyNodeServerName, metav1.GetOptions{}, + ) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err == nil { + if !args.Declarative { + return nil + } - _, err = k8s.KubeClient().AppsV1().DaemonSets(namespace).Create( - ctx, daemonset, metav1.CreateOptions{}, - ) - if err != nil { - if apierrors.IsAlreadyExists(err) { - err = nil + if daemonset.Spec.Selector != nil && daemonset.Spec.Selector.MatchLabels != nil { + selectorValue = daemonset.Spec.Selector.MatchLabels[selectorKey] + } } - return err + } + if selectorValue == "" { + selectorValue = fmt.Sprintf("%v-%v", consts.Identity, getRandSuffix()) } - _, err = io.WriteString(args.auditWriter, utils.MustGetYAML(daemonset)) - return err -} + daemonset := newDaemonset(podSpec, consts.LegacyNodeServerName, selectorValue, args.AppArmorProfile) -func createDaemonset(ctx context.Context, args *Args) (err error) { - if args.dryRun() { - if err := doCreateDaemonset(ctx, args); err != nil { + if !args.DryRun && !args.Declarative { + _, err = k8s.KubeClient().AppsV1().DaemonSets(namespace).Create( + ctx, daemonset, metav1.CreateOptions{}, + ) + if err != nil && !apierrors.IsAlreadyExists(err) { return err } + } - if args.Legacy { - if err := doCreateLegacyDaemonset(ctx, args); err != nil { - return err - } - } + return args.writeObject(daemonset) +} - return nil - } +func createDaemonset(ctx context.Context, args *Args) (err error) { if !sendProgressMessage(ctx, args.ProgressCh, fmt.Sprintf("Creating %s Daemonset", consts.NodeServerName), 1, nil) { return errSendProgress } - _, err = k8s.KubeClient().AppsV1().DaemonSets(namespace).Get( - ctx, consts.NodeServerName, metav1.GetOptions{}, - ) - if err != nil { - if !apierrors.IsNotFound(err) { - return err - } - if err := doCreateDaemonset(ctx, args); err != nil { - return err - } + if err := doCreateDaemonset(ctx, args); err != nil { + return err } if !sendProgressMessage(ctx, args.ProgressCh, fmt.Sprintf("Created %s Daemonset", consts.NodeServerName), 1, daemonsetComponent(consts.NodeServerName)) { return errSendProgress } + if !args.Legacy { return nil } + if !sendProgressMessage(ctx, args.ProgressCh, fmt.Sprintf("Creating %s Daemonset", consts.LegacyNodeServerName), 2, nil) { return errSendProgress } - _, err = k8s.KubeClient().AppsV1().DaemonSets(namespace).Get( - ctx, consts.LegacyNodeServerName, metav1.GetOptions{}, - ) - if err != nil { - if !apierrors.IsNotFound(err) { - return err - } - if err := doCreateLegacyDaemonset(ctx, args); err != nil { - return err - } + if err := doCreateLegacyDaemonset(ctx, args); err != nil { + return err } if !sendProgressMessage(ctx, args.ProgressCh, fmt.Sprintf("Created %s Daemonset", consts.LegacyNodeServerName), 2, daemonsetComponent(consts.LegacyNodeServerName)) { return errSendProgress } + return nil } diff --git a/pkg/installer/deployment.go b/pkg/installer/deployment.go index ca9c5b848..5c0b56ae5 100644 --- a/pkg/installer/deployment.go +++ b/pkg/installer/deployment.go @@ -19,11 +19,9 @@ package installer import ( "context" "fmt" - "io" "github.com/minio/directpv/pkg/consts" "github.com/minio/directpv/pkg/k8s" - "github.com/minio/directpv/pkg/utils" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -178,7 +176,24 @@ func doCreateDeployment(ctx context.Context, args *Args, legacy bool, step int) }, } - selectorValue := fmt.Sprintf("%v-%v", consts.ControllerServerName, getRandSuffix()) + var selectorValue string + if !args.DryRun { + deployment, err := k8s.KubeClient().AppsV1().Deployments(namespace).Get( + ctx, name, metav1.GetOptions{}, + ) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err == nil { + if deployment.Spec.Selector != nil && deployment.Spec.Selector.MatchLabels != nil { + selectorValue = deployment.Spec.Selector.MatchLabels[selectorKey] + } + } + } + if selectorValue == "" { + selectorValue = fmt.Sprintf("%v-%v", consts.ControllerServerName, getRandSuffix()) + } + replicas := int32(3) deployment := &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ @@ -208,27 +223,21 @@ func doCreateDeployment(ctx context.Context, args *Args, legacy bool, step int) }, Spec: podSpec, }, + Strategy: appsv1.DeploymentStrategy{Type: appsv1.RecreateDeploymentStrategyType}, }, Status: appsv1.DeploymentStatus{}, } - if args.dryRun() { - args.DryRunPrinter(deployment) - return nil - } - - _, err = k8s.KubeClient().AppsV1().Deployments(namespace).Create( - ctx, deployment, metav1.CreateOptions{}, - ) - if err != nil { - if apierrors.IsAlreadyExists(err) { - err = nil + if !args.DryRun && !args.Declarative { + _, err = k8s.KubeClient().AppsV1().Deployments(namespace).Create( + ctx, deployment, metav1.CreateOptions{}, + ) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err } - return err } - _, err = io.WriteString(args.auditWriter, utils.MustGetYAML(deployment)) - return err + return args.writeObject(deployment) } func createDeployment(ctx context.Context, args *Args) (err error) { diff --git a/pkg/installer/installer.go b/pkg/installer/installer.go index 68af4dc0e..f461fd5f8 100644 --- a/pkg/installer/installer.go +++ b/pkg/installer/installer.go @@ -85,7 +85,7 @@ func Install(ctx context.Context, args *Args) (err error) { } switch { - case args.dryRun(): + case args.DryRun: if args.KubeVersion == nil { // default higher version if args.KubeVersion, err = version.ParseSemantic("1.27.0"); err != nil { @@ -113,7 +113,7 @@ func Install(ctx context.Context, args *Args) (err error) { if args.KubeVersion.Major() != 1 || args.KubeVersion.Minor() < 18 || args.KubeVersion.Minor() > 27 { - if !args.dryRun() { + if !args.DryRun { utils.Eprintf( args.Quiet, false, diff --git a/pkg/installer/installer_test.go b/pkg/installer/installer_test.go index 8efe980a6..bdbae0d01 100644 --- a/pkg/installer/installer_test.go +++ b/pkg/installer/installer_test.go @@ -142,8 +142,8 @@ func TestGetKubeVersion(t *testing.T) { func TestInstallUinstall(t *testing.T) { args := Args{ - image: "directpv-0.0.0dev0", - auditWriter: io.Discard, + image: "directpv-0.0.0dev0", + ObjectWriter: io.Discard, } testVersions := []version.Info{ diff --git a/pkg/installer/migrate.go b/pkg/installer/migrate.go index 4f8b20a45..1d9ca82e5 100644 --- a/pkg/installer/migrate.go +++ b/pkg/installer/migrate.go @@ -19,6 +19,7 @@ package installer import ( "context" "fmt" + "os" "regexp" "strings" @@ -70,14 +71,14 @@ func (migrateTask) End(ctx context.Context, args *Args, err error) error { } func (migrateTask) Execute(ctx context.Context, args *Args) error { - return Migrate(ctx, args) + return Migrate(ctx, args, true) } func (migrateTask) Delete(_ context.Context, _ *Args) error { return nil } -func migrateDrives(ctx context.Context, dryRun bool, progressCh chan<- Message) (driveMap map[string]string, legacyDriveErrors map[string]error, driveErrors map[string]error, err error) { +func migrateDrives(ctx context.Context, dryRun bool, progressCh chan<- Message) (driveMap map[string]string, legacyDriveErrors, driveErrors map[string]error, err error) { ctx, cancelFunc := context.WithCancel(ctx) defer cancelFunc() @@ -221,7 +222,7 @@ func migrateDrives(ctx context.Context, dryRun bool, progressCh chan<- Message) return driveMap, legacyDriveErrors, driveErrors, nil } -func migrateVolumes(ctx context.Context, driveMap map[string]string, dryRun bool, progressCh chan<- Message) (legacyVolumeErrors map[string]error, volumeErrors map[string]error, err error) { +func migrateVolumes(ctx context.Context, driveMap map[string]string, dryRun bool, progressCh chan<- Message) (legacyVolumeErrors, volumeErrors map[string]error, err error) { ctx, cancelFunc := context.WithCancel(ctx) defer cancelFunc() @@ -334,8 +335,8 @@ func migrateVolumes(ctx context.Context, driveMap map[string]string, dryRun bool } // Migrate migrates legacy drives and volumes. -func Migrate(ctx context.Context, args *Args) (err error) { - if args.dryRun() || !args.Legacy { +func Migrate(ctx context.Context, args *Args, installer bool) (err error) { + if (installer && args.DryRun) || args.Declarative || !args.Legacy { return nil } @@ -363,14 +364,14 @@ func Migrate(ctx context.Context, args *Args) (err error) { return fmt.Errorf("migration does not support DirectCSIVolume version %v", version) } - driveMap, legacyDriveErrors, driveErrors, err := migrateDrives(ctx, args.dryRun(), args.ProgressCh) + driveMap, legacyDriveErrors, driveErrors, err := migrateDrives(ctx, args.DryRun, args.ProgressCh) if err != nil { return err } if !sendProgressMessage(ctx, args.ProgressCh, "Migrated the drives", 1, nil) { return errSendProgress } - legacyVolumeErrors, volumeErrors, err := migrateVolumes(ctx, driveMap, args.dryRun(), args.ProgressCh) + legacyVolumeErrors, volumeErrors, err := migrateVolumes(ctx, driveMap, args.DryRun, args.ProgressCh) if err != nil { return err } @@ -404,3 +405,85 @@ func Migrate(ctx context.Context, args *Args) (err error) { return nil } + +// RemoveLegacyDrives removes legacy drive CRDs. +func RemoveLegacyDrives(ctx context.Context, backupFile string) (backupCreated bool, err error) { + var drives []directv1beta5.DirectCSIDrive + for result := range legacyclient.ListDrives(ctx) { + if result.Err != nil { + return false, fmt.Errorf("unable to get legacy drives; %w", result.Err) + } + drives = append(drives, result.Drive) + } + if len(drives) == 0 { + return false, nil + } + + data, err := utils.ToYAML(directv1beta5.DirectCSIDriveList{ + TypeMeta: metav1.TypeMeta{ + Kind: "List", + APIVersion: "v1", + }, + Items: drives, + }) + if err != nil { + return false, fmt.Errorf("unable to generate legacy drives YAML; %w", err) + } + + if err = os.WriteFile(backupFile, data, os.ModePerm); err != nil { + return false, fmt.Errorf("unable to write legacy drives YAML; %w", err) + } + + for _, drive := range drives { + drive.Finalizers = []string{} + if _, err := legacyclient.DriveClient().Update(ctx, &drive, metav1.UpdateOptions{}); err != nil { + return false, fmt.Errorf("unable to update legacy drive %v; %w", drive.Name, err) + } + if err := legacyclient.DriveClient().Delete(ctx, drive.Name, metav1.DeleteOptions{}); err != nil { + return false, fmt.Errorf("unable to remove legacy drive %v; %w", drive.Name, err) + } + } + + return true, nil +} + +// RemoveLegacyVolumes removes legacy volume CRDs. +func RemoveLegacyVolumes(ctx context.Context, backupFile string) (backupCreated bool, err error) { + var volumes []directv1beta5.DirectCSIVolume + for result := range legacyclient.ListVolumes(ctx) { + if result.Err != nil { + return false, fmt.Errorf("unable to get legacy volumes; %w", result.Err) + } + volumes = append(volumes, result.Volume) + } + if len(volumes) == 0 { + return false, nil + } + + data, err := utils.ToYAML(directv1beta5.DirectCSIVolumeList{ + TypeMeta: metav1.TypeMeta{ + Kind: "List", + APIVersion: "v1", + }, + Items: volumes, + }) + if err != nil { + return false, fmt.Errorf("unable to generate legacy volumes YAML; %w", err) + } + + if err = os.WriteFile(backupFile, data, os.ModePerm); err != nil { + return false, fmt.Errorf("unable to write legacy volumes YAML; %w", err) + } + + for _, volume := range volumes { + volume.Finalizers = nil + if _, err := legacyclient.VolumeClient().Update(ctx, &volume, metav1.UpdateOptions{}); err != nil { + return false, fmt.Errorf("unable to update legacy volume %v; %w", volume.Name, err) + } + if err := legacyclient.VolumeClient().Delete(ctx, volume.Name, metav1.DeleteOptions{}); err != nil { + return false, fmt.Errorf("unable to remove legacy volume %v; %w", volume.Name, err) + } + } + + return true, nil +} diff --git a/pkg/installer/namespace.go b/pkg/installer/namespace.go index 697e59281..e0e81a051 100644 --- a/pkg/installer/namespace.go +++ b/pkg/installer/namespace.go @@ -18,10 +18,8 @@ package installer import ( "context" - "io" "github.com/minio/directpv/pkg/k8s" - "github.com/minio/directpv/pkg/utils" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -97,20 +95,14 @@ func createNamespace(ctx context.Context, args *Args) (err error) { }, } - if args.dryRun() { - args.DryRunPrinter(ns) - return nil - } - _, err = k8s.KubeClient().CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) - if err != nil { - if apierrors.IsAlreadyExists(err) { - err = nil + if !args.DryRun && !args.Declarative { + _, err = k8s.KubeClient().CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err } - return err } - _, err = io.WriteString(args.auditWriter, utils.MustGetYAML(ns)) - return err + return args.writeObject(ns) } func deleteNamespace(ctx context.Context) error { diff --git a/pkg/installer/psp.go b/pkg/installer/psp.go index 926eae00d..430009819 100644 --- a/pkg/installer/psp.go +++ b/pkg/installer/psp.go @@ -19,11 +19,9 @@ package installer import ( "context" "errors" - "io" "github.com/minio/directpv/pkg/consts" "github.com/minio/directpv/pkg/k8s" - "github.com/minio/directpv/pkg/utils" corev1 "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1beta1" rbac "k8s.io/api/rbac/v1" @@ -108,21 +106,14 @@ func createPSPClusterRoleBinding(ctx context.Context, args *Args) (err error) { }, } - if args.dryRun() { - args.DryRunPrinter(crb) - return nil - } - - _, err = k8s.KubeClient().RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}) - if err != nil { - if apierrors.IsAlreadyExists(err) { - err = nil + if !args.DryRun && !args.Declarative { + _, err = k8s.KubeClient().RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err } - return err } - _, err = io.WriteString(args.auditWriter, utils.MustGetYAML(crb)) - return err + return args.writeObject(crb) } func createPodSecurityPolicy(ctx context.Context, args *Args) (err error) { @@ -176,23 +167,16 @@ func createPodSecurityPolicy(ctx context.Context, args *Args) (err error) { }, } - if args.dryRun() { - args.DryRunPrinter(psp) - return nil - } - - _, err = k8s.KubeClient().PolicyV1beta1().PodSecurityPolicies().Create( - ctx, psp, metav1.CreateOptions{}, - ) - if err != nil { - if apierrors.IsAlreadyExists(err) { - err = nil + if !args.DryRun && !args.Declarative { + _, err = k8s.KubeClient().PolicyV1beta1().PodSecurityPolicies().Create( + ctx, psp, metav1.CreateOptions{}, + ) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err } - return err } - _, err = io.WriteString(args.auditWriter, utils.MustGetYAML(psp)) - return err + return args.writeObject(psp) } func createPSP(ctx context.Context, args *Args) error { @@ -200,7 +184,7 @@ func createPSP(ctx context.Context, args *Args) error { return nil } var gvk *schema.GroupVersionKind - if !args.dryRun() { + if !args.DryRun { var err error if gvk, err = k8s.GetGroupVersionKind("policy", "PodSecurityPolicy", "v1beta1"); err != nil { return err diff --git a/pkg/installer/rbac.go b/pkg/installer/rbac.go index cf06a2b17..ea34ff3e9 100644 --- a/pkg/installer/rbac.go +++ b/pkg/installer/rbac.go @@ -18,11 +18,9 @@ package installer import ( "context" - "io" "github.com/minio/directpv/pkg/consts" "github.com/minio/directpv/pkg/k8s" - "github.com/minio/directpv/pkg/utils" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -106,23 +104,16 @@ func createServiceAccount(ctx context.Context, args *Args) (err error) { AutomountServiceAccountToken: nil, } - if args.dryRun() { - args.DryRunPrinter(serviceAccount) - return nil - } - - _, err = k8s.KubeClient().CoreV1().ServiceAccounts(namespace).Create( - ctx, serviceAccount, metav1.CreateOptions{}, - ) - if err != nil { - if apierrors.IsAlreadyExists(err) { - err = nil + if !args.DryRun && !args.Declarative { + _, err = k8s.KubeClient().CoreV1().ServiceAccounts(namespace).Create( + ctx, serviceAccount, metav1.CreateOptions{}, + ) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err } - return err } - _, err = io.WriteString(args.auditWriter, utils.MustGetYAML(serviceAccount)) - return err + return args.writeObject(serviceAccount) } func createClusterRole(ctx context.Context, args *Args) (err error) { @@ -179,23 +170,16 @@ func createClusterRole(ctx context.Context, args *Args) (err error) { AggregationRule: nil, } - if args.dryRun() { - args.DryRunPrinter(clusterRole) - return nil - } - - _, err = k8s.KubeClient().RbacV1().ClusterRoles().Create( - ctx, clusterRole, metav1.CreateOptions{}, - ) - if err != nil { - if apierrors.IsAlreadyExists(err) { - err = nil + if !args.DryRun && !args.Declarative { + _, err = k8s.KubeClient().RbacV1().ClusterRoles().Create( + ctx, clusterRole, metav1.CreateOptions{}, + ) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err } - return err } - _, err = io.WriteString(args.auditWriter, utils.MustGetYAML(clusterRole)) - return err + return args.writeObject(clusterRole) } func createClusterRoleBinding(ctx context.Context, args *Args) (err error) { @@ -236,23 +220,16 @@ func createClusterRoleBinding(ctx context.Context, args *Args) (err error) { }, } - if args.dryRun() { - args.DryRunPrinter(clusterRoleBinding) - return nil - } - - _, err = k8s.KubeClient().RbacV1().ClusterRoleBindings().Create( - ctx, clusterRoleBinding, metav1.CreateOptions{}, - ) - if err != nil { - if apierrors.IsAlreadyExists(err) { - err = nil + if !args.DryRun && !args.Declarative { + _, err = k8s.KubeClient().RbacV1().ClusterRoleBindings().Create( + ctx, clusterRoleBinding, metav1.CreateOptions{}, + ) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err } - return err } - _, err = io.WriteString(args.auditWriter, utils.MustGetYAML(clusterRoleBinding)) - return err + return args.writeObject(clusterRoleBinding) } func createRole(ctx context.Context, args *Args) (err error) { @@ -284,23 +261,16 @@ func createRole(ctx context.Context, args *Args) (err error) { }, } - if args.dryRun() { - args.DryRunPrinter(role) - return nil - } - - _, err = k8s.KubeClient().RbacV1().Roles(namespace).Create( - ctx, role, metav1.CreateOptions{}, - ) - if err != nil { - if apierrors.IsAlreadyExists(err) { - err = nil + if !args.DryRun && !args.Declarative { + _, err = k8s.KubeClient().RbacV1().Roles(namespace).Create( + ctx, role, metav1.CreateOptions{}, + ) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err } - return err } - _, err = io.WriteString(args.auditWriter, utils.MustGetYAML(role)) - return err + return args.writeObject(role) } func createRoleBinding(ctx context.Context, args *Args) (err error) { @@ -341,23 +311,16 @@ func createRoleBinding(ctx context.Context, args *Args) (err error) { }, } - if args.dryRun() { - args.DryRunPrinter(roleBinding) - return nil - } - - _, err = k8s.KubeClient().RbacV1().RoleBindings(namespace).Create( - ctx, roleBinding, metav1.CreateOptions{}, - ) - if err != nil { - if apierrors.IsAlreadyExists(err) { - err = nil + if !args.DryRun && !args.Declarative { + _, err = k8s.KubeClient().RbacV1().RoleBindings(namespace).Create( + ctx, roleBinding, metav1.CreateOptions{}, + ) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err } - return err } - _, err = io.WriteString(args.auditWriter, utils.MustGetYAML(roleBinding)) - return err + return args.writeObject(roleBinding) } func createRBAC(ctx context.Context, args *Args) (err error) { diff --git a/pkg/installer/storageclass.go b/pkg/installer/storageclass.go index c87616016..73d697217 100644 --- a/pkg/installer/storageclass.go +++ b/pkg/installer/storageclass.go @@ -20,13 +20,11 @@ import ( "context" "errors" "fmt" - "io" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" "github.com/minio/directpv/pkg/consts" "github.com/minio/directpv/pkg/k8s" legacyclient "github.com/minio/directpv/pkg/legacy/client" - "github.com/minio/directpv/pkg/utils" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" storagev1beta1 "k8s.io/api/storage/v1beta1" @@ -117,23 +115,16 @@ func doCreateStorageClass(ctx context.Context, args *Args, version string, legac Parameters: map[string]string{"fstype": "xfs"}, } - if args.dryRun() { - args.DryRunPrinter(storageClass) - return nil - } - - _, err := k8s.KubeClient().StorageV1().StorageClasses().Create( - ctx, storageClass, metav1.CreateOptions{}, - ) - if err != nil { - if apierrors.IsAlreadyExists(err) { - err = nil + if !args.DryRun && !args.Declarative { + _, err := k8s.KubeClient().StorageV1().StorageClasses().Create( + ctx, storageClass, metav1.CreateOptions{}, + ) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err } - return err } - _, err = io.WriteString(args.auditWriter, utils.MustGetYAML(storageClass)) - return err + return args.writeObject(storageClass) case "v1beta1": bindingMode := storagev1beta1.VolumeBindingWaitForFirstConsumer @@ -156,23 +147,16 @@ func doCreateStorageClass(ctx context.Context, args *Args, version string, legac Parameters: map[string]string{"fstype": "xfs"}, } - if args.dryRun() { - args.DryRunPrinter(storageClass) - return nil - } - - _, err := k8s.KubeClient().StorageV1beta1().StorageClasses().Create( - ctx, storageClass, metav1.CreateOptions{}, - ) - if err != nil { - if apierrors.IsAlreadyExists(err) { - err = nil + if !args.DryRun && !args.Declarative { + _, err := k8s.KubeClient().StorageV1beta1().StorageClasses().Create( + ctx, storageClass, metav1.CreateOptions{}, + ) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err } - return err } - _, err = io.WriteString(args.auditWriter, utils.MustGetYAML(storageClass)) - return err + return args.writeObject(storageClass) default: return errStorageClassVersionUnsupported @@ -182,7 +166,7 @@ func doCreateStorageClass(ctx context.Context, args *Args, version string, legac func createStorageClass(ctx context.Context, args *Args) (err error) { version := "v1" switch { - case args.dryRun(): + case args.DryRun: if args.KubeVersion.Major() >= 1 && args.KubeVersion.Minor() < 16 { version = "v1beta1" } diff --git a/pkg/installer/utils.go b/pkg/installer/utils.go index 2abf3ea7f..c37ecaab3 100644 --- a/pkg/installer/utils.go +++ b/pkg/installer/utils.go @@ -20,8 +20,6 @@ import ( "context" "crypto/rand" "encoding/base32" - "fmt" - "io" "path" "strings" @@ -186,17 +184,14 @@ func migrateLog(ctx context.Context, args *Args, errMsg string, showInProgress b return errSendProgress } } - case !args.Quiet && !args.dryRun(): + case !args.Quiet && !args.DryRun: klog.Error(errMsg) } - return writeToAuditFile(args.auditWriter, errMsg) -} -func writeToAuditFile(writer io.Writer, message string) error { - if writer == nil { - return nil + if args.ObjectWriter != nil { + _, err := args.ObjectWriter.Write([]byte(errMsg)) + return err } - log := fmt.Sprintf("\n%s\n---\n", message) - _, err := io.WriteString(writer, log) - return err + + return nil } diff --git a/pkg/legacy/client/list.go b/pkg/legacy/client/list.go index c709deb4f..66b6ccede 100644 --- a/pkg/legacy/client/list.go +++ b/pkg/legacy/client/list.go @@ -56,11 +56,8 @@ func ListDrives(ctx context.Context) <-chan ListDriveResult { } for _, item := range result.Items { - switch item.Status.DriveStatus { - case directv1beta5.DriveStatusReady, directv1beta5.DriveStatusInUse: - if !send(ListDriveResult{Drive: item}) { - return - } + if !send(ListDriveResult{Drive: item}) { + return } } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 90157d1c7..c206a298d 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -19,13 +19,11 @@ package utils import ( "encoding/json" "fmt" - "io" "os" "path/filepath" "strings" "github.com/fatih/color" - "k8s.io/klog/v2" "sigs.k8s.io/yaml" ) @@ -40,31 +38,19 @@ func Contains[ctype comparable](slice []ctype, value ctype) bool { return false } -// MustGetYAML converts the given object to YAML -func MustGetYAML(i interface{}) string { +// ToYAML converts any type to YAML +func ToYAML(i interface{}) ([]byte, error) { data, err := yaml.Marshal(i) if err != nil { - klog.Fatalf("unable to marshal object to YAML; %w", err) + return nil, err } - return fmt.Sprintf("%v\n---\n", string(data)) -} -// MustGetJSON converts the given object to JSON -func MustGetJSON(obj interface{}) string { - data, err := json.MarshalIndent(obj, "", " ") - if err != nil { - klog.Fatalf("unable to marshal object to JSON; %w", err) - } - return fmt.Sprintf("%v\n---\n", string(data)) + return append(data, []byte("\n---\n")...), nil } -// WriteObject writes the writer content -func WriteObject(writer io.Writer, obj interface{}) error { - if _, err := writer.Write([]byte(MustGetYAML(obj))); err != nil { - return err - } - _, err := writer.Write([]byte("---\n")) - return err +// ToJSON converts any type to JSON +func ToJSON(obj interface{}) ([]byte, error) { + return json.MarshalIndent(obj, "", " ") } // SafeFile is used to write the yaml diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go deleted file mode 100644 index fd4417fc8..000000000 --- a/pkg/utils/utils_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// This file is part of MinIO DirectPV -// Copyright (c) 2021, 2022 MinIO, Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package utils - -import ( - "bytes" - "testing" - - "github.com/minio/directpv/pkg/consts" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestWriteObject(t *testing.T) { - var byteBuffer bytes.Buffer - meta := metav1.ObjectMeta{ - Name: consts.GroupName, - Namespace: consts.GroupName, - Annotations: map[string]string{ - consts.GroupName + "/created-by": "kubectl/directpv", - }, - Labels: map[string]string{ - "app": consts.GroupName, - "type": "CSIDriver", - }, - } - testCases := []struct { - input interface{} - errReturned bool - }{ - { - input: &corev1.Namespace{ - TypeMeta: metav1.TypeMeta{ - Kind: "Namespace", - APIVersion: "v1", - }, - ObjectMeta: meta, - Spec: corev1.NamespaceSpec{ - Finalizers: []corev1.FinalizerName{}, - }, - Status: corev1.NamespaceStatus{}, - }, - errReturned: false, - }, - {[]string{"1"}, false}, - } - for i, test := range testCases { - err := WriteObject(&byteBuffer, testCases[i].input) - errReturned := err != nil - if errReturned != test.errReturned { - t.Fatalf("Test %d: expected %t got %t", i+1, test.errReturned, errReturned) - } - } -}