From 16ca134e476f2218795c0b5f2414a7935199dee1 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Thu, 2 Oct 2025 13:22:40 +0300 Subject: [PATCH 01/16] terraform test: Try applying orchestratord version --- test/terraform/mzcompose.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/terraform/mzcompose.py b/test/terraform/mzcompose.py index 896995d67a248..e4412e8582d2e 100644 --- a/test/terraform/mzcompose.py +++ b/test/terraform/mzcompose.py @@ -290,6 +290,16 @@ def kubectl_setup( ], cwd=self.path, ) + spawn.runv( + [ + "kubectl", + "describe", + "-n", + "materialize", + pod_name, + ], + cwd=self.path, + ) status = spawn.capture( [ "kubectl", From c25bd9324bd675c573a567637220cd8c9a30a41f Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Thu, 2 Oct 2025 13:52:47 +0300 Subject: [PATCH 02/16] terraform test: override operator image tag --- test/terraform/azure-temporary/main.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/terraform/azure-temporary/main.tf b/test/terraform/azure-temporary/main.tf index d90fa0bbe72e3..3909b666dfd7d 100644 --- a/test/terraform/azure-temporary/main.tf +++ b/test/terraform/azure-temporary/main.tf @@ -84,6 +84,9 @@ module "materialize" { # Remove this when that is fixed. swap_enabled = false } + image = { + tag = var.operator_version + } }, } From 3cdef2e99fc300e6eee4a8c5ab4a6683697887eb Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Thu, 2 Oct 2025 16:32:52 +0300 Subject: [PATCH 03/16] terraform test: override default helm values --- test/terraform/azure-temporary/main.tf | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/test/terraform/azure-temporary/main.tf b/test/terraform/azure-temporary/main.tf index 3909b666dfd7d..2c9906a646d11 100644 --- a/test/terraform/azure-temporary/main.tf +++ b/test/terraform/azure-temporary/main.tf @@ -87,7 +87,28 @@ module "materialize" { image = { tag = var.operator_version } - }, + cloudProvider = { + type = "azure" + region = "eastus2" + } + } + observability = { + podMetrics = { + enabled = true + } + } + storage = { + storageClass = { + create = true + name = "openebs-lvm-instance-store-ext4" + provisioner = "local.csi.openebs.io" + parameters = { + storage = "lvm" + fsType = "ext4" + volgroup = "instance-store-vg" + } + } + } } materialize_instances = var.materialize_instances From d89b88193741424ded040e67d79b6a03df840cde Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Thu, 2 Oct 2025 18:05:43 +0300 Subject: [PATCH 04/16] terraform test: use correct version in helm yaml --- test/terraform/azure-temporary/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/terraform/azure-temporary/main.tf b/test/terraform/azure-temporary/main.tf index 2c9906a646d11..d1a3c4d069ffa 100644 --- a/test/terraform/azure-temporary/main.tf +++ b/test/terraform/azure-temporary/main.tf @@ -85,7 +85,7 @@ module "materialize" { swap_enabled = false } image = { - tag = var.operator_version + tag = var.orchestratord_version } cloudProvider = { type = "azure" From 86a84f00dd30addda11248a8e9aa4dd998e0b913 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Thu, 2 Oct 2025 20:07:24 +0300 Subject: [PATCH 05/16] terraform test: add more debugging info --- test/terraform/mzcompose.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/terraform/mzcompose.py b/test/terraform/mzcompose.py index e4412e8582d2e..71c7826380dbb 100644 --- a/test/terraform/mzcompose.py +++ b/test/terraform/mzcompose.py @@ -385,6 +385,31 @@ def kubectl_setup( ], cwd=self.path, ) + pod_name = spawn.capturev( + [ + "kubectl", + "get", + "pods", + "-n", + "materialize-environment", + "-o", + "jsonpath={.items[*].metadata.name}", + ], + cwd=self.path, + ).strip() + envd_pods = [p for p in pod_name.split() if "environmentd" in p] + for pod in envd_pods: + spawn.runv( + [ + "kubectl", + "describe", + "pod", + pod, + "-n", + "materialize-environment", + ], + cwd=self.path, + ) break except subprocess.CalledProcessError: time.sleep(1) From 2b26f67cfe124719ad275d9456d5efe88e095a90 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Thu, 2 Oct 2025 22:00:16 +0300 Subject: [PATCH 06/16] terraform test: fix typo in workflow --- test/terraform/mzcompose.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/terraform/mzcompose.py b/test/terraform/mzcompose.py index 71c7826380dbb..9ac289c4d5ba9 100644 --- a/test/terraform/mzcompose.py +++ b/test/terraform/mzcompose.py @@ -385,7 +385,7 @@ def kubectl_setup( ], cwd=self.path, ) - pod_name = spawn.capturev( + pod_name = spawn.capture( [ "kubectl", "get", From f0fadbc08054bef24bfdd3addc902d49c01ebf2c Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Fri, 3 Oct 2025 10:55:16 +0300 Subject: [PATCH 07/16] terraform test: fix typo in workflow --- test/terraform/mzcompose.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/test/terraform/mzcompose.py b/test/terraform/mzcompose.py index 9ac289c4d5ba9..e4412e8582d2e 100644 --- a/test/terraform/mzcompose.py +++ b/test/terraform/mzcompose.py @@ -385,31 +385,6 @@ def kubectl_setup( ], cwd=self.path, ) - pod_name = spawn.capture( - [ - "kubectl", - "get", - "pods", - "-n", - "materialize-environment", - "-o", - "jsonpath={.items[*].metadata.name}", - ], - cwd=self.path, - ).strip() - envd_pods = [p for p in pod_name.split() if "environmentd" in p] - for pod in envd_pods: - spawn.runv( - [ - "kubectl", - "describe", - "pod", - pod, - "-n", - "materialize-environment", - ], - cwd=self.path, - ) break except subprocess.CalledProcessError: time.sleep(1) From 1816e737a5a539b6a002099b0900f7240521748b Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Fri, 3 Oct 2025 10:57:08 +0300 Subject: [PATCH 08/16] terraform test: add more debugging info --- test/terraform/mzcompose.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/terraform/mzcompose.py b/test/terraform/mzcompose.py index e4412e8582d2e..6d7b5be7650ea 100644 --- a/test/terraform/mzcompose.py +++ b/test/terraform/mzcompose.py @@ -396,6 +396,42 @@ def kubectl_setup( ["kubectl", "get", "pods", "-n", "materialize-environment"], cwd=self.path, ) + pod_name = spawn.capture( + [ + "kubectl", + "get", + "pods", + "-n", + "materialize-environment", + "-o", + "jsonpath={.items[*].metadata.name}", + ], + cwd=self.path, + ).strip() + envd_pods = [p for p in pod_name.split() if "environmentd" in p] + for pod in envd_pods: + spawn.runv( + [ + "kubectl", + "describe", + "pod", + pod, + "-n", + "materialize-environment", + ], + cwd=self.path, + ) + spawn.runv( + [ + "kubectl", + "describe", + "pod", + pod, + "-n", + "materialize-environment", + ], + cwd=self.path, + ) status = spawn.capture( [ "kubectl", From e5d0a47b1a132a4fc9466eb0d63f4688014fdbaf Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Fri, 3 Oct 2025 12:44:10 +0300 Subject: [PATCH 09/16] terraform test: add a small delay --- test/terraform/mzcompose.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/test/terraform/mzcompose.py b/test/terraform/mzcompose.py index 6d7b5be7650ea..0c9ff7080f672 100644 --- a/test/terraform/mzcompose.py +++ b/test/terraform/mzcompose.py @@ -290,16 +290,6 @@ def kubectl_setup( ], cwd=self.path, ) - spawn.runv( - [ - "kubectl", - "describe", - "-n", - "materialize", - pod_name, - ], - cwd=self.path, - ) status = spawn.capture( [ "kubectl", @@ -392,11 +382,12 @@ def kubectl_setup( raise ValueError("Never completed") for i in range(240): try: + time.sleep(2) spawn.runv( ["kubectl", "get", "pods", "-n", "materialize-environment"], cwd=self.path, ) - pod_name = spawn.capture( + pod_names = spawn.capture( [ "kubectl", "get", @@ -408,7 +399,7 @@ def kubectl_setup( ], cwd=self.path, ).strip() - envd_pods = [p for p in pod_name.split() if "environmentd" in p] + envd_pods = [p for p in pod_names.split() if "environmentd" in p] for pod in envd_pods: spawn.runv( [ From 62236c4fe3837c82c0fb21f9e006623eeb4cd6a0 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Fri, 3 Oct 2025 15:46:41 +0300 Subject: [PATCH 10/16] terraform test: add logs output --- test/terraform/mzcompose.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/terraform/mzcompose.py b/test/terraform/mzcompose.py index 0c9ff7080f672..af8d181cf46db 100644 --- a/test/terraform/mzcompose.py +++ b/test/terraform/mzcompose.py @@ -412,6 +412,17 @@ def kubectl_setup( ], cwd=self.path, ) + # Get the logs of the pod: + spawn.runv( + [ + "kubectl", + "logs", + pod, + "-n", + "materialize-environment", + ], + cwd=self.path, + ) spawn.runv( [ "kubectl", From 731e30f34d489e12ea10d376d03e41e7939c2923 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Fri, 3 Oct 2025 22:09:25 +0300 Subject: [PATCH 11/16] terraform test: add 120s sleep for envd to boot --- test/terraform/mzcompose.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/terraform/mzcompose.py b/test/terraform/mzcompose.py index af8d181cf46db..950ffd3e45ccb 100644 --- a/test/terraform/mzcompose.py +++ b/test/terraform/mzcompose.py @@ -380,6 +380,9 @@ def kubectl_setup( time.sleep(1) else: raise ValueError("Never completed") + sleep_time = 120 + print(f"Sleeping for {sleep_time} seconds to give the pods a chance to start and then checking:") + time.sleep(sleep_time) for i in range(240): try: time.sleep(2) From e0c801592450360718d921633bcc73f86fd20b6f Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Fri, 3 Oct 2025 23:04:59 +0300 Subject: [PATCH 12/16] terraform test: reduce to 30s sleep for envd to boot --- test/terraform/mzcompose.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/terraform/mzcompose.py b/test/terraform/mzcompose.py index 950ffd3e45ccb..ebcebd6ee3810 100644 --- a/test/terraform/mzcompose.py +++ b/test/terraform/mzcompose.py @@ -380,7 +380,7 @@ def kubectl_setup( time.sleep(1) else: raise ValueError("Never completed") - sleep_time = 120 + sleep_time = 30 print(f"Sleeping for {sleep_time} seconds to give the pods a chance to start and then checking:") time.sleep(sleep_time) for i in range(240): From af378925f4407a6a4bf7679f452922b8f61f5831 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Fri, 3 Oct 2025 23:10:20 +0300 Subject: [PATCH 13/16] terraform test: select only first balancerd pod --- test/terraform/mzcompose.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/terraform/mzcompose.py b/test/terraform/mzcompose.py index ebcebd6ee3810..850e28675d7a7 100644 --- a/test/terraform/mzcompose.py +++ b/test/terraform/mzcompose.py @@ -616,7 +616,7 @@ def connect(self, c: Composition) -> None: "jsonpath={.items[*].metadata.name}", ], cwd=self.path, - ) + ).strip()[0] # error: arguments in resource/name form must have a single resource and name print(f"Got balancerd name: {balancerd_name}") From 91396e18bb6639a4d53bd450aefdaac030989bd7 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Sat, 4 Oct 2025 00:48:32 +0300 Subject: [PATCH 14/16] terraform test: fix lint --- test/terraform/mzcompose.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/terraform/mzcompose.py b/test/terraform/mzcompose.py index 850e28675d7a7..7fc0125e0b52e 100644 --- a/test/terraform/mzcompose.py +++ b/test/terraform/mzcompose.py @@ -380,9 +380,8 @@ def kubectl_setup( time.sleep(1) else: raise ValueError("Never completed") - sleep_time = 30 - print(f"Sleeping for {sleep_time} seconds to give the pods a chance to start and then checking:") - time.sleep(sleep_time) + # Give the pods a chance to start and then check: + time.sleep(30) for i in range(240): try: time.sleep(2) From b11e5f72d55b0d79d91748cc53e46689d58af224 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Sat, 4 Oct 2025 00:54:08 +0300 Subject: [PATCH 15/16] terraform test: select only first balancerd pod --- test/terraform/mzcompose.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/terraform/mzcompose.py b/test/terraform/mzcompose.py index 7fc0125e0b52e..50af10fa54224 100644 --- a/test/terraform/mzcompose.py +++ b/test/terraform/mzcompose.py @@ -612,10 +612,10 @@ def connect(self, c: Composition) -> None: "-n", "materialize-environment", "-o", - "jsonpath={.items[*].metadata.name}", + "jsonpath={.items[0].metadata.name}", ], cwd=self.path, - ).strip()[0] + ).strip() # error: arguments in resource/name form must have a single resource and name print(f"Got balancerd name: {balancerd_name}") From ce011273ce71945f50478f7c283e1d4a96394554 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Sat, 4 Oct 2025 10:29:07 +0300 Subject: [PATCH 16/16] terraform test: remove unused pod describe --- test/terraform/mzcompose.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/test/terraform/mzcompose.py b/test/terraform/mzcompose.py index 50af10fa54224..90c8e7b0c5e96 100644 --- a/test/terraform/mzcompose.py +++ b/test/terraform/mzcompose.py @@ -425,17 +425,6 @@ def kubectl_setup( ], cwd=self.path, ) - spawn.runv( - [ - "kubectl", - "describe", - "pod", - pod, - "-n", - "materialize-environment", - ], - cwd=self.path, - ) status = spawn.capture( [ "kubectl",