Add GCS E2E

Signed-off-by: d-kuro <kurosawa7620@gmail.com>

e2e/Makefile

@@ -42,6 +42,7 @@ endif
 	$(KUSTOMIZE) build . | $(KUBECTL) apply -f -
 	$(KUBECTL) -n moco-system wait --for=condition=available --timeout=180s --all deployments
 	$(KUBECTL) apply -f minio.yaml
+	$(KUBECTL) apply -f fake-gcs-server.yaml
 	$(KUBECTL) wait --timeout=60s --for=condition=Ready --all pods
 
 .PHONY: test

e2e/backup_gcs_test.go

@@ -0,0 +1,164 @@
+package e2e
+
+import (
+	"bytes"
+	_ "embed"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"text/template"
+	"time"
+
+	mocov1beta2 "github.com/cybozu-go/moco/api/v1beta2"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+)
+
+//go:embed testdata/backup_gcs.yaml
+var backupGCSYAML string
+
+//go:embed testdata/restore_gcs.yaml
+var restoreGCSYAML string
+
+var _ = Context("backup", func() {
+	if doUpgrade {
+		return
+	}
+
+	var restorePoint time.Time
+
+	It("should construct a source cluster", func() {
+		kubectlSafe(fillTemplate(backupGCSYAML), "apply", "-f", "-")
+		Eventually(func() error {
+			cluster, err := getCluster("backup", "source")
+			if err != nil {
+				return err
+			}
+			for _, cond := range cluster.Status.Conditions {
+				if cond.Type != mocov1beta2.ConditionHealthy {
+					continue
+				}
+				if cond.Status == corev1.ConditionTrue {
+					return nil
+				}
+				return fmt.Errorf("cluster is not healthy: %s", cond.Status)
+			}
+			return errors.New("no health condition")
+		}).Should(Succeed())
+
+		kubectlSafe(nil, "moco", "-n", "backup", "mysql", "-u", "moco-writable", "source", "--",
+			"-e", "CREATE DATABASE test")
+		kubectlSafe(nil, "moco", "-n", "backup", "mysql", "-u", "moco-writable", "source", "--",
+			"-D", "test", "-e", "CREATE TABLE t (id INT NOT NULL AUTO_INCREMENT, data VARCHAR(32) NOT NULL, PRIMARY KEY (id), KEY key1 (data), KEY key2 (data, id)) ENGINE=InnoDB")
+		kubectlSafe(nil, "moco", "-n", "backup", "mysql", "-u", "moco-writable", "source", "--",
+			"-D", "test", "--init_command=SET autocommit=1", "-e", "INSERT INTO t (data) VALUES ('aaa')")
+	})
+
+	It("should take a full dump", func() {
+		kubectlSafe(nil, "-n", "backup", "create", "job", "--from=cronjob/moco-backup-source", "backup-1")
+		Eventually(func() error {
+			out, err := kubectl(nil, "-n", "backup", "get", "jobs", "backup-1", "-o", "json")
+			if err != nil {
+				return err
+			}
+			job := &batchv1.Job{}
+			if err := json.Unmarshal(out, job); err != nil {
+				return err
+			}
+			for _, cond := range job.Status.Conditions {
+				if cond.Type != batchv1.JobComplete {
+					continue
+				}
+				if cond.Status == corev1.ConditionTrue {
+					return nil
+				}
+			}
+			return errors.New("backup-1 has not been finished")
+		}).Should(Succeed())
+	})
+
+	It("should take an incremental backup", func() {
+		kubectlSafe(nil, "moco", "-n", "backup", "mysql", "-u", "moco-writable", "source", "--",
+			"-D", "test", "--init_command=SET autocommit=1", "-e", "INSERT INTO t (data) VALUES ('bbb')")
+		time.Sleep(1100 * time.Millisecond)
+		restorePoint = time.Now().UTC()
+		time.Sleep(1100 * time.Millisecond)
+		kubectlSafe(nil, "moco", "-n", "backup", "mysql", "-u", "moco-admin", "source", "--",
+			"-D", "test", "--init_command=SET autocommit=1", "-e", "FLUSH LOCAL BINARY LOGS")
+		kubectlSafe(nil, "moco", "-n", "backup", "mysql", "-u", "moco-writable", "source", "--",
+			"-D", "test", "--init_command=SET autocommit=1", "-e", "INSERT INTO t (data) VALUES ('ccc')")
+		time.Sleep(100 * time.Millisecond)
+
+		kubectlSafe(nil, "-n", "backup", "create", "job", "--from=cronjob/moco-backup-source", "backup-2")
+		Eventually(func() error {
+			out, err := kubectl(nil, "-n", "backup", "get", "jobs", "backup-2", "-o", "json")
+			if err != nil {
+				return err
+			}
+			job := &batchv1.Job{}
+			if err := json.Unmarshal(out, job); err != nil {
+				return err
+			}
+			for _, cond := range job.Status.Conditions {
+				if cond.Type != batchv1.JobComplete {
+					continue
+				}
+				if cond.Status == corev1.ConditionTrue {
+					return nil
+				}
+			}
+			return errors.New("backup-2 has not been finished")
+		}).Should(Succeed())
+
+		cluster, err := getCluster("backup", "source")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(cluster.Status.Backup.BinlogSize).NotTo(Equal(int64(0)))
+	})
+
+	It("should destroy the source then restore the backup data", func() {
+		kubectlSafe(nil, "-n", "backup", "delete", "mysqlclusters", "source")
+
+		tmpl, err := template.New("").Parse(restoreGCSYAML)
+		Expect(err).NotTo(HaveOccurred())
+		buf := new(bytes.Buffer)
+		err = tmpl.Execute(buf, struct {
+			MySQLVersion string
+			RestorePoint string
+		}{
+			mysqlVersion,
+			restorePoint.Format(time.RFC3339),
+		})
+		Expect(err).NotTo(HaveOccurred())
+
+		kubectlSafe(buf.Bytes(), "apply", "-f", "-")
+		Eventually(func() error {
+			cluster, err := getCluster("backup", "target")
+			if err != nil {
+				return err
+			}
+			for _, cond := range cluster.Status.Conditions {
+				if cond.Type != mocov1beta2.ConditionHealthy {
+					continue
+				}
+				if cond.Status == corev1.ConditionTrue {
+					return nil
+				}
+			}
+			return errors.New("target is not healthy")
+		}).Should(Succeed())
+
+		out := kubectlSafe(nil, "moco", "-n", "backup", "mysql", "target", "--",
+			"-N", "-D", "test", "-e", "SELECT COUNT(*) FROM t")
+		count, err := strconv.Atoi(strings.TrimSpace(string(out)))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(count).To(Equal(2))
+	})
+
+	It("should delete clusters", func() {
+		kubectlSafe(nil, "delete", "-n", "backup", "mysqlclusters", "--all")
+	})
+})

e2e/fake-gcs-server.yaml

@@ -0,0 +1,39 @@
+apiVersion: v1
+kind: Service
+metadata:
+  namespace: default
+  name: fake-gcs-server
+spec:
+  ports:
+    - name: fake-gcs-server
+      port: 4443
+      targetPort: fake-gcs-server
+      protocol: TCP
+  selector:
+    name: fake-gcs-server
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: default
+  name: fake-gcs-server
+  labels:
+    name: fake-gcs-server
+spec:
+  containers:
+    - name: fake-gcs-server
+      image: fsouza/fake-gcs-server
+      args:
+        - "-scheme=http"
+        - "-port=4443"
+        - "-public-host=fake-gcs-server.default.svc:4443"
+      ports:
+        - name: fake-gcs-server
+          containerPort: 4443
+          protocol: TCP
+      volumeMounts:
+        - name: bucket
+          mountPath: /data/moco
+  volumes:
+    - name: bucket
+      emptyDir: {}

e2e/testdata/backup_gcs.yaml

@@ -0,0 +1,78 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: backup-gcs
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: backup-gcs
+  name: mycnf
+data:
+  innodb_log_file_size: "10M"
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: backup-gcs
+  name: backup-owner
+---
+apiVersion: moco.cybozu.com/v1beta1
+kind: BackupPolicy
+metadata:
+  namespace: backup-gcs
+  name: daily
+spec:
+  schedule: "@daily"
+  jobConfig:
+    serviceAccountName: backup-owner
+    env:
+    - name: STORAGE_EMULATOR_HOST
+      value: fake-gcs-server.default.svc:4443
+    bucketConfig:
+      bucketName: moco
+      endpointURL: http://fake-gcs-server.default.svc:4443
+      backendType: gcs
+    workVolume:
+      emptyDir: {}
+---
+apiVersion: moco.cybozu.com/v1beta2
+kind: MySQLCluster
+metadata:
+  namespace: backup-gcs
+  name: source
+spec:
+  mysqlConfigMapName: mycnf
+  replicas: 3
+  backupPolicyName: daily
+  podTemplate:
+    spec:
+      containers:
+      - name: mysqld
+        image: quay.io/cybozu/mysql:{{ . }}
+    # Specify minimum resources so as not to overwhelm CI resources.
+    overwriteContainers:
+      - name: agent
+        resources:
+          requests:
+            cpu: 1m
+      - name: moco-init
+        resources:
+          requests:
+            cpu: 1m
+      - name: slow-log
+        resources:
+          requests:
+            cpu: 1m
+      - name: mysqld-exporter
+        resources:
+          requests:
+            cpu: 1m
+  volumeClaimTemplates:
+  - metadata:
+      name: mysql-data
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 1Gi

e2e/testdata/restore_gcs.yaml

@@ -0,0 +1,54 @@
+apiVersion: moco.cybozu.com/v1beta2
+kind: MySQLCluster
+metadata:
+  namespace: backup-gcs
+  name: target
+spec:
+  mysqlConfigMapName: mycnf
+  replicas: 1
+  restore:
+    sourceName: source
+    sourceNamespace: backup
+    restorePoint: "{{ .RestorePoint }}"
+    jobConfig:
+      serviceAccountName: backup-owner
+      env:
+      - name: STORAGE_EMULATOR_HOST
+        value: fake-gcs-server.default.svc:4443
+      bucketConfig:
+        bucketName: moco
+        endpointURL: http://fake-gcs-server.default.svc:4443
+        backendType: gcs
+      workVolume:
+        emptyDir: {}
+  podTemplate:
+    spec:
+      containers:
+      - name: mysqld
+        image: quay.io/cybozu/mysql:{{ .MySQLVersion }}
+    # Specify minimum resources so as not to overwhelm CI resources.
+    overwriteContainers:
+      - name: agent
+        resources:
+          requests:
+            cpu: 1m
+      - name: moco-init
+        resources:
+          requests:
+            cpu: 1m
+      - name: slow-log
+        resources:
+          requests:
+            cpu: 1m
+      - name: mysqld-exporter
+        resources:
+          requests:
+            cpu: 1m
+  volumeClaimTemplates:
+  - metadata:
+      name: mysql-data
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 1Gi