diff --git a/reboot.go b/reboot.go index 862b1dcb7..37a88953c 100644 --- a/reboot.go +++ b/reboot.go @@ -361,7 +361,7 @@ func createArangoDeployment(cli acli.Interface, ns, deplname, arangoimage string for _, info := range agnt { depl.Status.Members.Agents = append(depl.Status.Members.Agents, deplv1alpha.MemberStatus{ - ID: info.UUID, + ID: info.UUID, PersistentVolumeClaimName: info.Claim, PodName: k8sutil.CreatePodName(deplname, deplv1alpha.ServerGroupAgents.AsRole(), info.UUID, "-rbt"), }) @@ -369,7 +369,7 @@ func createArangoDeployment(cli acli.Interface, ns, deplname, arangoimage string for _, info := range prmr { depl.Status.Members.DBServers = append(depl.Status.Members.DBServers, deplv1alpha.MemberStatus{ - ID: info.UUID, + ID: info.UUID, PersistentVolumeClaimName: info.Claim, PodName: k8sutil.CreatePodName(deplname, deplv1alpha.ServerGroupDBServers.AsRole(), info.UUID, "-rbt"), }) diff --git a/tests/sidecar_test.go b/tests/sidecar_test.go index a2c4a2a91..9e498897b 100644 --- a/tests/sidecar_test.go +++ b/tests/sidecar_test.go @@ -25,6 +25,7 @@ import ( "context" "fmt" "testing" + "time" driver "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" @@ -148,22 +149,24 @@ func runSideCarTest(t *testing.T, spec SideCarTest) { var name = "nginx" var image = "nginx:1.7.9" - spec.AddSideCar(coordinators, v1.Container{Image: image, Name: name}) - deployment, err = updateDeployment(c, depl.GetName(), ns, - func(depl *api.DeploymentSpec) { - depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) - }) - if err != nil { - t.Fatalf("Failed to add %s to group %s", name, coordinators) - } else { - t.Logf("Add %s sidecar to group %s ...", name, coordinators) - } - err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) - if err != nil { - t.Fatalf("... failed: %v", err) - } else { - t.Log("... done") - } + t.Run("Add first sidecar to coordinators", func(t *testing.T) { + spec.AddSideCar(coordinators, v1.Container{Image: image, Name: name}) + deployment, err = updateDeployment(c, depl.GetName(), ns, + func(depl *api.DeploymentSpec) { + depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) + }) + if err != nil { + t.Fatalf("Failed to add %s to group %s", name, coordinators) + } else { + t.Logf("Add %s sidecar to group %s ...", name, coordinators) + } + err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) + if err != nil { + t.Fatalf("... failed: %v", err) + } else { + t.Log("... done") + } + }) cmd1 := []string{"sh", "-c", "sleep 3600"} cmd2 := []string{"sh", "-c", "sleep 1800"} @@ -173,213 +176,247 @@ func runSideCarTest(t *testing.T, spec SideCarTest) { // Add 2nd sidecar to coordinators image = "busybox" name = "sleeper" - spec.AddSideCar(coordinators, v1.Container{Image: image, Name: name, Command: cmd1}) - deployment, err = updateDeployment(c, depl.GetName(), ns, - func(depl *api.DeploymentSpec) { - depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) - }) - if err != nil { - t.Fatalf("Failed to add %s to group %s", name, coordinators) - } else { - t.Logf("Add sidecar %s to group %s ...", name, coordinators) - } - err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) - if err != nil { - t.Fatalf("... failed: %v", err) - } else { - t.Log("... done") - } + t.Run("Add 2nd sidecar to coordinators", func(t *testing.T) { + spec.AddSideCar(coordinators, v1.Container{Image: image, Name: name, Command: cmd1}) + deployment, err = updateDeployment(c, depl.GetName(), ns, + func(depl *api.DeploymentSpec) { + depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) + }) + if err != nil { + t.Fatalf("Failed to add %s to group %s", name, coordinators) + } else { + t.Logf("Add sidecar %s to group %s ...", name, coordinators) + } + err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) + if err != nil { + t.Fatalf("... failed: %v", err) + } else { + t.Log("... done") + } + }) // Update command line of second sidecar - spec.GroupSideCars(coordinators)[1].Command = cmd2 - deployment, err = updateDeployment(c, depl.GetName(), ns, - func(depl *api.DeploymentSpec) { - depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) - }) - if err != nil { - t.Fatalf("Failed to update %s in group %s with new command line", name, coordinators) - } else { - t.Logf("Update %s in group %s with new command line ...", name, coordinators) - } - err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) - if err != nil { - t.Fatalf("... failed: %v", err) - } else { - t.Log("... done") - } + t.Run("Update command line of second sidecar", func(t *testing.T) { + spec.GroupSideCars(coordinators)[1].Command = cmd2 + deployment, err = updateDeployment(c, depl.GetName(), ns, + func(depl *api.DeploymentSpec) { + depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) + }) + if err != nil { + t.Fatalf("Failed to update %s in group %s with new command line", name, coordinators) + } else { + t.Logf("Update %s in group %s with new command line ...", name, coordinators) + } + err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) + if err != nil { + t.Fatalf("... failed: %v", err) + } else { + t.Log("... done") + } + }) // Change command line args of second sidecar - spec.GroupSideCars(coordinators)[1].Command = cmd - spec.GroupSideCars(coordinators)[1].Args = args - deployment, err = updateDeployment(c, depl.GetName(), ns, - func(depl *api.DeploymentSpec) { - depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) - }) - if err != nil { - t.Fatalf("Failed to update %s in group %s with new command line arguments", name, coordinators) - } else { - t.Logf("Update %s in group %s with new command line arguments ...", name, coordinators) - } - err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) - if err != nil { - t.Fatalf("... failed: %v", err) - } else { - t.Log("... done") - } + t.Run("Update command line args of second sidecar", func(t *testing.T) { + spec.GroupSideCars(coordinators)[1].Command = cmd + spec.GroupSideCars(coordinators)[1].Args = args + deployment, err = updateDeployment(c, depl.GetName(), ns, + func(depl *api.DeploymentSpec) { + depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) + }) + if err != nil { + t.Fatalf("Failed to update %s in group %s with new command line arguments", name, coordinators) + } else { + t.Logf("Update %s in group %s with new command line arguments ...", name, coordinators) + } + err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) + if err != nil { + t.Fatalf("... failed: %v", err) + } else { + t.Log("... done") + } + }) // Change environment variables of second container - spec.GroupSideCars(coordinators)[1].Env = []v1.EnvVar{ - {Name: "Hello", Value: "World"}, {Name: "Pi", Value: "3.14159265359"}, {Name: "Two", Value: "2"}} - deployment, err = updateDeployment(c, depl.GetName(), ns, - func(depl *api.DeploymentSpec) { - depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) - }) - if err != nil { - t.Fatalf("Failed to change environment variables of %s sidecars for %s", name, coordinators) - } else { - t.Logf("Change environment variables of %s sidecars for %s ...", name, coordinators) - } - err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) - if err != nil { - t.Fatalf("... failed: %v", err) - } else { - t.Log("... done") - } + t.Run("Change environment variables of second sidecar", func(t *testing.T) { + spec.GroupSideCars(coordinators)[1].Env = []v1.EnvVar{ + {Name: "Hello", Value: "World"}, {Name: "Pi", Value: "3.14159265359"}, {Name: "Two", Value: "2"}} + deployment, err = updateDeployment(c, depl.GetName(), ns, + func(depl *api.DeploymentSpec) { + depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) + }) + if err != nil { + t.Fatalf("Failed to change environment variables of %s sidecars for %s", name, coordinators) + } else { + t.Logf("Change environment variables of %s sidecars for %s ...", name, coordinators) + } + err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) + if err != nil { + t.Fatalf("... failed: %v", err) + } else { + t.Log("... done") + } + }) // Upgrade side car image name = spec.GroupSideCars(coordinators)[0].Name - spec.GroupSideCars(coordinators)[0].Image = "nginx:1.7.10" - deployment, err = updateDeployment(c, depl.GetName(), ns, - func(depl *api.DeploymentSpec) { - depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) - }) - if err != nil { - t.Fatalf("Failed to update %s in group %s with new image", name, coordinators) - } else { - t.Logf("Update image of sidecar %s in group %s ...", name, coordinators) - } - err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) - if err != nil { - t.Fatalf("... failed: %v", err) - } else { - t.Log("... done") - } + t.Run("Upgrade side car image", func(t *testing.T) { + spec.GroupSideCars(coordinators)[0].Image = "nginx:1.7.10" + deployment, err = updateDeployment(c, depl.GetName(), ns, + func(depl *api.DeploymentSpec) { + depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) + }) + if err != nil { + t.Fatalf("Failed to update %s in group %s with new image", name, coordinators) + } else { + t.Logf("Update image of sidecar %s in group %s ...", name, coordinators) + } + err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) + if err != nil { + t.Fatalf("... failed: %v", err) + } else { + t.Log("... done") + } + }) // Update side car image with new pull policy - spec.GroupSideCars(coordinators)[0].ImagePullPolicy = v1.PullPolicy("Always") - deployment, err = updateDeployment(c, depl.GetName(), ns, - func(depl *api.DeploymentSpec) { - depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) - }) - if err != nil { - t.Fatalf("Failed to update %s in group %s with new image pull policy", name, coordinators) - } else { - t.Logf("Update %s in group %s with new image pull policy ...", name, coordinators) - } - err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) - if err != nil { - t.Fatalf("... failed: %v", err) - } else { - t.Log("... done") - } + t.Run("Update side car image with new pull policy", func(t *testing.T) { + spec.GroupSideCars(coordinators)[0].ImagePullPolicy = v1.PullPolicy("Always") + deployment, err = updateDeployment(c, depl.GetName(), ns, + func(depl *api.DeploymentSpec) { + depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) + }) + if err != nil { + t.Fatalf("Failed to update %s in group %s with new image pull policy", name, coordinators) + } else { + t.Logf("Update %s in group %s with new image pull policy ...", name, coordinators) + } + err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) + if err != nil { + t.Fatalf("... failed: %v", err) + } else { + t.Log("... done") + } + }) // Remove all sidecars again - spec.ClearGroupSideCars(coordinators) - deployment, err = updateDeployment(c, depl.GetName(), ns, - func(depl *api.DeploymentSpec) { - depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) - }) - if err != nil { - t.Fatalf("Failed to remove all sidecars from group %s", coordinators) - } else { - t.Logf("Remove all sidecars from group %s ...", coordinators) - } - err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) - if err != nil { - t.Fatalf("... failed: %v", err) - } else { - t.Log("... done") - } + t.Run("Remove all sidecars again", func(t *testing.T) { + spec.ClearGroupSideCars(coordinators) + deployment, err = updateDeployment(c, depl.GetName(), ns, + func(depl *api.DeploymentSpec) { + depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) + }) + if err != nil { + t.Fatalf("Failed to remove all sidecars from group %s", coordinators) + } else { + t.Logf("Remove all sidecars from group %s ...", coordinators) + } + err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) + if err != nil { + t.Fatalf("... failed: %v", err) + } else { + t.Log("... done") + } + }) // Adding containers to coordinators and db servers image = "busybox" name = "sleeper" - spec.AddSideCar(coordinators, v1.Container{Image: image, Name: name, Command: cmd1}) - spec.AddSideCar(dbservers, v1.Container{Image: image, Name: name, Command: cmd1}) - deployment, err = updateDeployment(c, depl.GetName(), ns, - func(depl *api.DeploymentSpec) { - depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) - depl.DBServers.Sidecars = spec.GroupSideCars(dbservers) - }) - if err != nil { - t.Fatalf("Failed to add a container to both coordinators and db servers") - } else { - t.Logf("Add %s sidecar to %s and %s ...", name, coordinators, dbservers) - } - err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) - if err != nil { - t.Fatalf("... failed: %v", err) - } else { - t.Log("... done") - } + t.Run("Add containers to coordinators and dbservers", func(t *testing.T) { + spec.AddSideCar(coordinators, v1.Container{Image: image, Name: name, Command: cmd1}) + spec.AddSideCar(dbservers, v1.Container{Image: image, Name: name, Command: cmd1}) + deployment, err = updateDeployment(c, depl.GetName(), ns, + func(depl *api.DeploymentSpec) { + depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) + depl.DBServers.Sidecars = spec.GroupSideCars(dbservers) + }) + if err != nil { + t.Fatalf("Failed to add a container to both coordinators and db servers") + } else { + t.Logf("Add %s sidecar to %s and %s ...", name, coordinators, dbservers) + } + err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) + if err != nil { + t.Fatalf("... failed: %v", err) + } else { + t.Log("... done") + } + }) + + // Check that no pod rotation happens for 2 mins, this is to check that + // no unnecessary rotations happen and to guard against a regression. + t.Run("Check no pod rotation", func(t *testing.T) { + d, err := waitUntilDeployment(c, depl.GetName(), ns, resourcesAsRequested(kubecli, ns)) + if err != nil { + t.Fatalf("Deployment not rotated in time: %s", err) + } + podCreationTimes := getPodCreationTimes(t, kubecli, d); + time.Sleep(2 * time.Minute) + checkPodCreationTimes(t, kubecli, d, podCreationTimes) + }) // Clear containers from both groups - spec.ClearGroupSideCars(coordinators) - spec.ClearGroupSideCars(dbservers) - deployment, err = updateDeployment(c, depl.GetName(), ns, - func(depl *api.DeploymentSpec) { - depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) - depl.DBServers.Sidecars = spec.GroupSideCars(dbservers) - }) - if err != nil { - t.Fatalf("Failed to delete all containers from both coordinators and db servers") - } else { - t.Logf("Remove all sidecars ...") - } - err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) - if err != nil { - t.Fatalf("... failed: %v", err) - } else { - t.Log("... done") - } + t.Run("Clear containers from both groups", func(t *testing.T) { + spec.ClearGroupSideCars(coordinators) + spec.ClearGroupSideCars(dbservers) + deployment, err = updateDeployment(c, depl.GetName(), ns, + func(depl *api.DeploymentSpec) { + depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) + depl.DBServers.Sidecars = spec.GroupSideCars(dbservers) + }) + if err != nil { + t.Fatalf("Failed to delete all containers from both coordinators and db servers") + } else { + t.Logf("Remove all sidecars ...") + } + err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) + if err != nil { + t.Fatalf("... failed: %v", err) + } else { + t.Log("... done") + } + }) - // Adding containers to agents again - spec.AddSideCar(agents, v1.Container{Image: image, Name: name, Command: cmd1}) - deployment, err = updateDeployment(c, depl.GetName(), ns, - func(depl *api.DeploymentSpec) { - depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) - }) - if err != nil { - t.Fatalf("Failed to add a %s sidecar to %s", name, agents) - } else { - t.Logf("Add a %s sidecar to %s ...", name, agents) - } - err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) - if err != nil { - t.Fatalf("... failed: %v", err) - } else { - t.Log("... done") - } + // Adding containers to coordinators again + t.Run("Add containers to coordinators again", func(t *testing.T) { + spec.AddSideCar(agents, v1.Container{Image: image, Name: name, Command: cmd1}) + deployment, err = updateDeployment(c, depl.GetName(), ns, + func(depl *api.DeploymentSpec) { + depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) + }) + if err != nil { + t.Fatalf("Failed to add a %s sidecar to %s", name, agents) + } else { + t.Logf("Add a %s sidecar to %s ...", name, agents) + } + err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) + if err != nil { + t.Fatalf("... failed: %v", err) + } else { + t.Log("... done") + } + }) // Clear containers from coordinators and add to db servers - spec.ClearGroupSideCars(agents) - spec.AddSideCar(dbservers, v1.Container{Image: image, Name: name, Command: cmd1}) - deployment, err = updateDeployment(c, depl.GetName(), ns, - func(depl *api.DeploymentSpec) { - depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) - depl.DBServers.Sidecars = spec.GroupSideCars(dbservers) - }) - if err != nil { - t.Fatalf("Failed to delete %s containers and add %s sidecars to %s", agents, name, dbservers) - } else { - t.Logf("Delete %s containers and add %s sidecars to %s", agents, name, dbservers) - } - err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) - if err != nil { - t.Fatalf("... failed: %v", err) - } else { - t.Log("... done") - } + t.Run("Clear containers from coordinators and add to dbservers", func(t *testing.T) { + spec.ClearGroupSideCars(agents) + spec.AddSideCar(dbservers, v1.Container{Image: image, Name: name, Command: cmd1}) + deployment, err = updateDeployment(c, depl.GetName(), ns, + func(depl *api.DeploymentSpec) { + depl.Coordinators.Sidecars = spec.GroupSideCars(coordinators) + depl.DBServers.Sidecars = spec.GroupSideCars(dbservers) + }) + if err != nil { + t.Fatalf("Failed to delete %s containers and add %s sidecars to %s", agents, name, dbservers) + } else { + t.Logf("Delete %s containers and add %s sidecars to %s", agents, name, dbservers) + } + err = waitUntilClusterSidecarsEqualSpec(t, spec.Mode(), *depl) + if err != nil { + t.Fatalf("... failed: %v", err) + } else { + t.Log("... done") + } + }) // Clean up removeDeployment(c, depl.GetName(), ns) diff --git a/tests/test_util.go b/tests/test_util.go index d4ce7bd5d..c73d3bd58 100644 --- a/tests/test_util.go +++ b/tests/test_util.go @@ -814,3 +814,40 @@ func testServerRole(ctx context.Context, client driver.Client, expectedRole driv } return nil } + +func getPodCreationTimes(t *testing.T, kubecli kubernetes.Interface, depl *api.ArangoDeployment) map[string]metav1.Time { + ns := getNamespace(t) + podCreationTimes := make(map[string]metav1.Time) + depl.ForeachServerGroup(func(group api.ServerGroup, spec api.ServerGroupSpec, status *api.MemberStatusList) error { + fmt.Printf("Looking at group %s with %d pods...\n", group.AsRole(), len(*status)) + for _, m := range *status { + // Get pod: + fmt.Printf("Looking at pod %s...\n", m.PodName) + pod, err := kubecli.CoreV1().Pods(ns).Get(m.PodName, metav1.GetOptions{}) + // Simply ignore error and skip pod: + if err == nil { + fmt.Printf("Found creation time of %v for pod %s\n", pod.GetCreationTimestamp(), m.PodName) + podCreationTimes[m.PodName] = pod.GetCreationTimestamp() + } else { + fmt.Printf("Could not get pod %s error: %v\n", m.PodName, err) + } + } + return nil + }, &depl.Status) + return podCreationTimes +} + +func checkPodCreationTimes(t *testing.T, kubecli kubernetes.Interface, depl *api.ArangoDeployment, times map[string]metav1.Time) { + foundTimes := getPodCreationTimes(t, kubecli, depl) + for name, timestamp := range(times) { + ti, found := foundTimes[name] + if !found { + t.Errorf("Did not find pod %s any more in creation time check!", name) + } else if ti != timestamp { + t.Errorf("Pod %s has been rotated unexpectedly in creation time check!", name) + } + } + if len(foundTimes) != len(times) { + t.Errorf("Number of pods found (%d) in creation time check does not match expected %d!", len(foundTimes), len(times)) + } +}