diff --git a/docker/Dockerfile b/docker/Dockerfile index 520fd2d07..4c47eb346 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,6 +2,7 @@ FROM alpine MAINTAINER Team ACID @ Zalando # We need root certificates to deal with teams api over https +RUN apk --no-cache add curl RUN apk --no-cache add ca-certificates COPY build/* / diff --git a/e2e/Makefile b/e2e/Makefile index a72c6bef0..b904ad763 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -51,3 +51,6 @@ tools: e2etest: tools copy clean ./run.sh main + +cleanup: clean + ./run.sh cleanup \ No newline at end of file diff --git a/e2e/README.md b/e2e/README.md index 92a1fc731..ce8931f62 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -33,7 +33,7 @@ runtime. In the e2e folder you can invoke tests either with `make test` or with: ```bash -./run.sh +./run.sh main ``` To run both the build and test step you can invoke `make e2e` from the parent @@ -41,7 +41,7 @@ directory. To run the end 2 end test and keep the kind state execute: ```bash -NOCLEANUP=True ./run.sh +NOCLEANUP=True ./run.sh main ``` ## Run indidual test diff --git a/e2e/scripts/watch_objects.sh b/e2e/scripts/watch_objects.sh index c866fbd45..9dde54f5e 100755 --- a/e2e/scripts/watch_objects.sh +++ b/e2e/scripts/watch_objects.sh @@ -13,7 +13,15 @@ kubectl get statefulsets echo kubectl get deployments echo +echo +echo 'Step from operator deployment' kubectl get pods -l name=postgres-operator -o jsonpath='{.items..metadata.annotations.step}' echo +echo +echo 'Spilo Image in statefulset' kubectl get pods -l application=spilo -o jsonpath='{.items..spec.containers..image}' -" \ No newline at end of file +echo +echo +echo 'Queue Status' +kubectl exec -it \$(kubectl get pods -l name=postgres-operator -o jsonpath='{.items..metadata.name}') -- curl localhost:8080/workers/all/status/ +echo" \ No newline at end of file diff --git a/e2e/tests/k8s_api.py b/e2e/tests/k8s_api.py index 371fa8e0d..f2abd8e0c 100644 --- a/e2e/tests/k8s_api.py +++ b/e2e/tests/k8s_api.py @@ -239,6 +239,19 @@ def get_patroni_state(self, pod): return [] return json.loads(r.stdout.decode()) + def get_operator_state(self): + pod = self.get_operator_pod() + if pod == None: + return None + pod = pod.metadata.name + + r = self.exec_with_kubectl(pod, "curl localhost:8080/workers/all/status/") + if not r.returncode == 0 or not r.stdout.decode()[0:1]=="{": + return None + + return json.loads(r.stdout.decode()) + + def get_patroni_running_members(self, pod="acid-minimal-cluster-0"): result = self.get_patroni_state(pod) return list(filter(lambda x: "State" in x and x["State"] == "running", result)) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 888fc2eaa..05cc09a70 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -109,9 +109,18 @@ def setUpClass(cls): with open("manifests/postgres-operator.yaml", 'w') as f: yaml.dump(operator_deployment, f, Dumper=yaml.Dumper) + with open("manifests/configmap.yaml", 'r+') as f: + configmap = yaml.safe_load(f) + configmap["data"]["workers"] = "1" + + with open("manifests/configmap.yaml", 'w') as f: + yaml.dump(configmap, f, Dumper=yaml.Dumper) + for filename in ["operator-service-account-rbac.yaml", + "postgresteam.crd.yaml", "configmap.yaml", "postgres-operator.yaml", + "api-service.yaml", "infrastructure-roles.yaml", "infrastructure-roles-new.yaml", "e2e-storage-class.yaml"]: @@ -338,6 +347,7 @@ def test_infrastructure_roles(self): }, } k8s.update_config(patch_infrastructure_roles) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") try: # check that new roles are represented in the config by requesting the @@ -447,6 +457,7 @@ def test_lazy_spilo_upgrade(self): # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod0), conf_image, "Rolling upgrade was not executed", 50, 3) self.eventuallyEqual(lambda: k8s.get_effective_pod_image(pod1), conf_image, "Rolling upgrade was not executed", 50, 3) + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), 2, "Postgres status did not enter running") except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) @@ -519,6 +530,9 @@ def get_docker_image(): print('Operator log: {}'.format(k8s.get_operator_log())) raise + # ensure cluster is healthy after tests + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running") + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_min_resource_limits(self): ''' @@ -809,12 +823,14 @@ def test_zzzz_cluster_deletion(self): } } k8s.update_config(patch_delete_annotations) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") try: # this delete attempt should be omitted because of missing annotations k8s.api.custom_objects_api.delete_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") time.sleep(5) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") # check that pods and services are still there k8s.wait_for_running_pods(cluster_label, 2) @@ -825,6 +841,7 @@ def test_zzzz_cluster_deletion(self): # wait a little before proceeding time.sleep(10) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") # add annotations to manifest delete_date = datetime.today().strftime('%Y-%m-%d') @@ -838,6 +855,7 @@ def test_zzzz_cluster_deletion(self): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_delete_annotations) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0":"idle"}, "Operator does not get in sync") # wait a little before proceeding time.sleep(20)