Add get, expand support for PVC's

Signed-off-by: Vineeth Pothulapati <vineethpothulapati@outlook.com>

cli/README.md

@@ -96,6 +96,17 @@ Documentation about Helm configuration can be found in the [Helm chart directory
 | `tobs metrics chunk-interval set`         | Sets the chunk interval of a specific metric to the specified duration.              | `--user`, `-U` : database user name <br> `--dbname`, `-d` : database name to connect to |
 | `tobs metrics chunk-interval reset`       | Resets chunk interval of a specific metric to the default value.                     | `--user`, `-U` : database user name <br> `--dbname`, `-d` : database name to connect to |
 
+### Volume Commands
+
+The volume operation is available for TimescaleDB & Prometheus PVC's.
+
+**Note**: To expand PVC's in Kubernetes cluster make sure you have configured `storageClass` with `allowVolumeExpansion: true` to allow PVC expansion.
+
+| Command                        | Description                                       | Flags                                |
+|--------------------------------|---------------------------------------------------|--------------------------------------|
+| `tobs volume get`     | Displays Persistent Volume Claims sizes. | `--timescaleDB-storage`, `s`, `--timescaleDB-wal`, `w`, `prometheus-storage`, `-p`  |
+| `tobs volume expand`  | Expands the Persistent Volume Claims for provided resources to specified sizes. The expansion size is allowed in `Ki`, `Mi` & `Gi` units. example: `150Gi`. | `--timescaleDB-storage`, `s`, `--timescaleDB-wal`, `w`, `prometheus-storage`, `-p` |
+
 ## Global Flags
 
 The following are global flags that can be used with any of the above commands:

cli/cmd/volume.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+)
+
+// volumeCmd represents the volume command
+var volumeCmd = &cobra.Command{
+	Use:   "volume",
+	Short: "Subcommand for Volume operations",
+}
+
+func init() {
+	rootCmd.AddCommand(volumeCmd)
+}

cli/cmd/volumeExpand.go

@@ -0,0 +1,83 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/timescale/tobs/cli/pkg/k8s"
+
+	"github.com/spf13/cobra"
+)
+
+// volumeExpandCmd represents the volume expand command
+var volumeExpandCmd = &cobra.Command{
+	Use:   "expand",
+	Short: "Expand PVC's",
+	Args:  cobra.ExactArgs(0),
+	RunE:  volumeExpand,
+}
+
+func init() {
+	volumeCmd.AddCommand(volumeExpandCmd)
+	volumeExpandCmd.Flags().StringP("timescaleDB-wal", "w", "", "Expand volume of timescaleDB wal")
+	volumeExpandCmd.Flags().StringP("timescaleDB-storage", "s", "", "Expand volume of timescaleDB storage")
+	volumeExpandCmd.Flags().StringP("prometheus-storage", "p", "", "Expand volume of prometheus storage")
+}
+
+func volumeExpand(cmd *cobra.Command, args []string) error {
+	tsDBWal, err := cmd.Flags().GetString("timescaleDB-wal")
+	if err != nil {
+		return fmt.Errorf("could not get timescaleDB-wal flag %w", err)
+	}
+
+	tsDBStorage, err := cmd.Flags().GetString("timescaleDB-storage")
+	if err != nil {
+		return fmt.Errorf("could not get timescaleDB-storage flag %w", err)
+	}
+
+	promStorage, err := cmd.Flags().GetString("prometheus-storage")
+	if err != nil {
+		return fmt.Errorf("could not get prometheus-storage flag %w", err)
+	}
+
+	if tsDBStorage != "" {
+		pvcPrefix := "storage-volume"
+		results, err := k8s.ExpandTimescaleDBPVC(namespace,  tsDBStorage, pvcPrefix , map[string]string{"app": name+"-timescaledb"})
+		if err != nil {
+			return fmt.Errorf("could not expand timescaleDB-storage: %w", err)
+		}
+		expandSuccessPrint(pvcPrefix, results)
+	}
+
+	if tsDBWal != "" {
+		pvcPrefix := "wal-volume"
+		results, err := k8s.ExpandTimescaleDBPVC(namespace, tsDBWal, "wal-volume", map[string]string{"app": name+"-timescaledb"})
+		if err != nil {
+			return fmt.Errorf("could not expand timescaleDB-wal: %w", err)
+		}
+
+		expandSuccessPrint(pvcPrefix, results)
+	}
+
+	if promStorage != "" {
+		pvcPrefix := name+"-prometheus-server"
+		err := k8s.ExpandPVC(namespace, pvcPrefix, promStorage)
+		if err != nil {
+			return fmt.Errorf("could not expand prometheus-storage: %w", err)
+		}
+
+		expandSuccessPrint(pvcPrefix, map[string]string{pvcPrefix: promStorage})
+	}
+
+	return nil
+}
+
+func expandSuccessPrint(pvcPrefix string, results map[string]string) {
+	if len(results) == 0 {
+		return
+	}
+
+	fmt.Printf("PVC's of %s\n", pvcPrefix)
+	for pvcName, value := range results {
+		fmt.Printf("Successfully expanded PVC: %s to %s\n", pvcName, value)
+	}
+	fmt.Println()
+}

cli/cmd/volumeGet.go

@@ -0,0 +1,80 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/timescale/tobs/cli/pkg/k8s"
+)
+
+// volumeGetCmd represents the volume expand command
+var volumeGetCmd = &cobra.Command{
+	Use:   "get",
+	Short: "Get PVC's volume",
+	Args:  cobra.ExactArgs(0),
+	RunE:  volumeGet,
+}
+
+func init() {
+	volumeCmd.AddCommand(volumeGetCmd)
+	volumeGetCmd.Flags().BoolP("timescaleDB-wal", "w", false, "Get volume of timescaleDB wal")
+	volumeGetCmd.Flags().BoolP("timescaleDB-storage", "s", false, "Get volume of timescaleDB storage")
+	volumeGetCmd.Flags().BoolP("prometheus-storage", "p", false, "Get volume of prometheus storage")
+}
+
+func volumeGet(cmd *cobra.Command, args []string) error {
+	tsDBWal, err := cmd.Flags().GetBool("timescaleDB-wal")
+	if err != nil {
+		return fmt.Errorf("could not get timescaleDB-wal flag %w", err)
+	}
+
+	tsDBStorage, err := cmd.Flags().GetBool("timescaleDB-storage")
+	if err != nil {
+		return fmt.Errorf("could not get timescaleDB-storage flag %w", err)
+	}
+
+	promStorage, err := cmd.Flags().GetBool("prometheus-storage")
+	if err != nil {
+		return fmt.Errorf("could not get prometheus-storage flag %w", err)
+	}
+
+	if tsDBStorage {
+		pvcPrefix := "storage-volume"
+		results, err := k8s.GetPVCSizes(namespace, pvcPrefix, map[string]string{"app": name+"-timescaledb"})
+		if err != nil {
+			return fmt.Errorf("could not get timescaleDB-storage: %w", err)
+		}
+		volumeGetPrint(pvcPrefix, results)
+	}
+
+	if tsDBWal {
+		pvcPrefix := "wal-volume"
+		results, err := k8s.GetPVCSizes(namespace, pvcPrefix, map[string]string{"app": name+"-timescaledb"})
+		if err != nil {
+			return fmt.Errorf("could not get timescaleDB-wal: %w", err)
+		}
+		volumeGetPrint(pvcPrefix, results)
+	}
+
+	if promStorage {
+		pvcPrefix := name+"-prometheus-server"
+		results, err := k8s.GetPVCSizes(namespace, pvcPrefix, nil)
+		if err != nil {
+			return fmt.Errorf("could not get prometheus-storage: %w", err)
+		}
+		volumeGetPrint(pvcPrefix, results)
+	}
+
+	return nil
+}
+
+func volumeGetPrint(pvcPrefix string, results map[string]string) {
+	if len(results) == 0 {
+		return
+	}
+
+	fmt.Printf("PVC's of %s\n", pvcPrefix)
+	for pvcName, value := range results {
+		fmt.Printf("Existing size of PVC: %s is %s\n", pvcName, value)
+	}
+	fmt.Println()
+}

cli/pkg/k8s/k8s.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"k8s.io/apimachinery/pkg/api/resource"
 	"log"
 	"net/http"
 	"os"
@@ -390,3 +391,109 @@ func KubeUpdateSecret(namespace string, secret *corev1.Secret) error {
 
 	return nil
 }
+
+func buildPVCNames(pvcPrefix string, pods []corev1.Pod) (pvcNames []string) {
+	for _, pod := range pods {
+		pvcNames = append(pvcNames, pvcPrefix+"-"+pod.Name)
+	}
+	return pvcNames
+}
+
+func GetPVCSizes(namespace, pvcPrefix string, labels map[string]string) (map[string]string, error) {
+	var pvcs []string
+	pvcSizes := make(map[string]string)
+	if labels != nil {
+		pods, err := KubeGetPods(namespace, labels)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get the pods using labels %w", err)
+		}
+		pvcs = buildPVCNames(pvcPrefix, pods)
+	} else {
+		pvcs = append(pvcs, pvcPrefix)
+	}
+
+	client, _ := kubeInit()
+	for _, pvcName := range pvcs {
+		podPVC, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(context.Background(), pvcName, metav1.GetOptions{})
+		if err != nil {
+			fmt.Println(fmt.Errorf("failed to get the pvc for %s %w", pvcName, err))
+		}
+
+		existingSize := podPVC.Spec.Resources.Requests["storage"]
+		pvcSizes[pvcName] = existingSize.String()
+	}
+
+	return pvcSizes, nil
+}
+
+func ExpandTimescaleDBPVC(namespace, value, pvcPrefix string, labels map[string]string) (map[string]string, error) {
+	pvcResults := make(map[string]string)
+	pods, err := KubeGetPods(namespace, labels)
+	if err != nil {
+		return pvcResults, fmt.Errorf("failed to get the pods using labels %w", err)
+	}
+
+	pvcs := buildPVCNames(pvcPrefix, pods)
+	for _, pvc := range pvcs {
+		err := ExpandPVC(namespace, pvc, value)
+		if err != nil {
+			fmt.Println(fmt.Errorf("%w",err))
+		} else {
+			pvcResults[pvc] = value
+		}
+	}
+	return pvcResults, nil
+}
+
+func ExpandPVC(namespace, pvcName, value string) error {
+	client, _ := kubeInit()
+	podPVC, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(context.Background(), pvcName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to get the pvc for %s %w", pvcName, err)
+	}
+
+	newSize, err := resource.ParseQuantity(value)
+	if err != nil {
+		return fmt.Errorf("failed to parse the volume size %w", err)
+	}
+
+	existingSize := podPVC.Spec.Resources.Requests["storage"]
+	if yes := newSize.Cmp(existingSize); yes != 1 {
+		return fmt.Errorf("provided volume size for pvc: %s is less than or equal to the existing size: %s", pvcName, existingSize.String())
+	}
+
+	podPVC.Spec.Resources.Requests["storage"] = newSize
+	_, err = client.CoreV1().PersistentVolumeClaims(namespace).Update(context.Background(), podPVC, metav1.UpdateOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to update persistent volume claim %w", err)
+	}
+
+	return nil
+}
+
+
+/*
+#########################################
+Kubernetes utils for e2e tests.
+#########################################
+ */
+
+// By default local storage provider doesn't let us to expand PVC's
+// For e2e tests to run we are configuring storageClass to allow PVC expansion
+func UpdateStorageClassAllowVolumeExpand() error {
+	client, _ := kubeInit()
+	storageClass, err := client.StorageV1().StorageClasses().Get(context.Background(), "standard", metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	setTrue := true
+	storageClass.AllowVolumeExpansion = &setTrue
+	_, err = client.StorageV1().StorageClasses().Update(context.Background(), storageClass, metav1.UpdateOptions{})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+

cli/tests/volume_test.go

@@ -0,0 +1,80 @@
+package tests
+
+import (
+	"github.com/timescale/tobs/cli/pkg/k8s"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+func testVolumeExpansion(t testing.TB, timescaleDBStorage, timescaleDBWal, prometheusStorage string) {
+	cmds := []string{"volume", "expand", "-n", RELEASE_NAME, "--namespace", NAMESPACE}
+	if timescaleDBStorage != "" {
+		cmds = append(cmds, "--timescaleDB-storage", timescaleDBStorage)
+	}
+
+	if timescaleDBWal != "" {
+		cmds = append(cmds, "--timescaleDB-wal", timescaleDBWal)
+	}
+
+	if prometheusStorage != "" {
+		cmds = append(cmds, "--prometheus-storage", prometheusStorage)
+	}
+
+	t.Logf("Running '%v'", "tobs "+strings.Join(cmds, " "))
+	 expand := exec.Command("./../bin/tobs", cmds...)
+	 out, err := expand.CombinedOutput()
+	 if err != nil {
+		 t.Logf(string(out))
+		 t.Fatal(err)
+	 }
+}
+
+func testVolumeGet(t testing.TB, timescaleDBStorage, timescaleDBWal, prometheusStorage bool) {
+	cmds := []string{"volume", "get", "-n", RELEASE_NAME, "--namespace", NAMESPACE}
+	if timescaleDBStorage {
+		cmds = append(cmds, "--timescaleDB-storage")
+	}
+
+	if timescaleDBWal {
+		cmds = append(cmds, "--timescaleDB-wal")
+	}
+
+	if prometheusStorage {
+		cmds = append(cmds, "--prometheus-storage")
+	}
+
+	t.Logf("Running '%v'", "tobs "+strings.Join(cmds, " "))
+	expand := exec.Command("./../bin/tobs", cmds...)
+	out, err := expand.CombinedOutput()
+	if err != nil {
+		t.Logf(string(out))
+		t.Fatal(err)
+	}
+}
+
+
+func TestVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping Prometheus tests")
+	}
+
+	testVolumeGet(t, true, true, true)
+	testVolumeGet(t, false, true, true)
+	testVolumeGet(t, true, true, false)
+	testVolumeGet(t, true, false, false)
+	testVolumeGet(t, false, false, true)
+
+	// update default storageClass in Kind to allow pvc expansion
+	err := k8s.UpdateStorageClassAllowVolumeExpand()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testVolumeExpansion(t, "151Gi", "21Gi", "9Gi")
+	testVolumeExpansion(t, "152Gi", "22Gi", "")
+	testVolumeExpansion(t, "153Gi", "", "")
+	testVolumeExpansion(t, "", "23Gi", "")
+	testVolumeExpansion(t, "", "24Gi", "10Gi")
+	testVolumeExpansion(t, "", "", "11Gi")
+}