-
Notifications
You must be signed in to change notification settings - Fork 162
/
scyllacluster_replace.go
114 lines (92 loc) · 4.17 KB
/
scyllacluster_replace.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
// Copyright (C) 2021 ScyllaDB
package scyllacluster
import (
"context"
"fmt"
"time"
g "github.com/onsi/ginkgo/v2"
o "github.com/onsi/gomega"
"github.com/scylladb/scylla-operator/pkg/naming"
scyllafixture "github.com/scylladb/scylla-operator/test/e2e/fixture/scylla"
"github.com/scylladb/scylla-operator/test/e2e/framework"
"github.com/scylladb/scylla-operator/test/e2e/utils"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
var _ = g.Describe("ScyllaCluster replace", func() {
defer g.GinkgoRecover()
f := framework.NewFramework("scyllacluster")
g.It("should replace a node", func() {
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
defer cancel()
sc := scyllafixture.BasicScyllaCluster.ReadOrFail()
sc.Spec.Datacenter.Racks[0].Members = 2
framework.By("Creating a ScyllaCluster")
sc, err := f.ScyllaClient().ScyllaV1().ScyllaClusters(f.Namespace()).Create(ctx, sc, metav1.CreateOptions{})
o.Expect(err).NotTo(o.HaveOccurred())
framework.By("Waiting for the ScyllaCluster to rollout (RV=%s)", sc.ResourceVersion)
waitCtx1, waitCtx1Cancel := utils.ContextForRollout(ctx, sc)
defer waitCtx1Cancel()
sc, err = utils.WaitForScyllaClusterState(waitCtx1, f.ScyllaClient().ScyllaV1(), sc.Namespace, sc.Name, utils.WaitForStateOptions{}, utils.IsScyllaClusterRolledOut)
o.Expect(err).NotTo(o.HaveOccurred())
verifyScyllaCluster(ctx, f.KubeClient(), sc)
hosts := getScyllaHostsAndWaitForFullQuorum(ctx, f.KubeClient().CoreV1(), sc)
o.Expect(hosts).To(o.HaveLen(2))
di := insertAndVerifyCQLData(ctx, hosts)
defer di.Close()
framework.By("Replacing a node #0")
pod, err := f.KubeClient().CoreV1().Pods(f.Namespace()).Get(
ctx,
utils.GetNodeName(sc, 0),
metav1.GetOptions{},
)
o.Expect(err).NotTo(o.HaveOccurred())
framework.Infof("Initial pod %q has UID is %q", pod.Name, pod.UID)
_, err = f.KubeClient().CoreV1().Services(f.Namespace()).Patch(
ctx,
pod.Name,
types.MergePatchType,
[]byte(fmt.Sprintf(
`{"metadata":{"labels": {"%s": ""}}}`,
naming.ReplaceLabel,
)),
metav1.PatchOptions{},
)
o.Expect(err).NotTo(o.HaveOccurred())
framework.By("Waiting for the pod to be replaced")
waitCtx2, waitCtx2Cancel := utils.ContextForRollout(ctx, sc)
defer waitCtx2Cancel()
_, err = utils.WaitForPodState(waitCtx2, f.KubeClient().CoreV1().Pods(pod.Namespace), pod.Name, utils.WaitForStateOptions{TolerateDelete: true}, func(p *corev1.Pod) (bool, error) {
return p.UID != pod.UID, nil
})
o.Expect(err).NotTo(o.HaveOccurred())
// Give the controller some time to observe that the pod is down.
time.Sleep(10 * time.Second)
framework.By("Waiting for the ScyllaCluster to re-deploy")
waitCtx3, waitCtx3Cancel := utils.ContextForRollout(ctx, sc)
defer waitCtx3Cancel()
sc, err = utils.WaitForScyllaClusterState(waitCtx3, f.ScyllaClient().ScyllaV1(), sc.Namespace, sc.Name, utils.WaitForStateOptions{}, utils.IsScyllaClusterRolledOut)
o.Expect(err).NotTo(o.HaveOccurred())
verifyScyllaCluster(ctx, f.KubeClient(), sc)
framework.By("Waiting for the other nodes to acknowledge the replace")
client, _, err := utils.GetScyllaClient(ctx, f.KubeClient().CoreV1(), sc)
o.Expect(err).NotTo(o.HaveOccurred())
replacedNodeService, err := f.KubeClient().CoreV1().Services(sc.Namespace).Get(ctx, utils.GetNodeName(sc, 0), metav1.GetOptions{})
o.Expect(err).NotTo(o.HaveOccurred())
otherNodeService, err := f.KubeClient().CoreV1().Services(sc.Namespace).Get(ctx, utils.GetNodeName(sc, 1), metav1.GetOptions{})
o.Expect(err).NotTo(o.HaveOccurred())
o.Eventually(func(g o.Gomega) {
status, err := client.Status(ctx, otherNodeService.Spec.ClusterIP)
g.Expect(err).NotTo(o.HaveOccurred())
g.Expect(status.LiveHosts()).To(o.ContainElement(replacedNodeService.Spec.ClusterIP))
}).WithPolling(time.Second).WithTimeout(5 * time.Minute).Should(o.Succeed())
oldHosts := hosts
hosts = getScyllaHostsAndWaitForFullQuorum(ctx, f.KubeClient().CoreV1(), sc)
o.Expect(hosts).To(o.HaveLen(len(oldHosts)))
o.Expect(hosts).NotTo(o.ConsistOf(oldHosts))
err = di.SetClientEndpoints(hosts)
o.Expect(err).NotTo(o.HaveOccurred())
verifyCQLData(ctx, di)
})
})