diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 1be783fc586..cc807c31234 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -430,7 +430,8 @@ false **/ItCrossDomainTransaction, - **/ItKubernetesEvents, + **/ItKubernetesDomainEvents, + **/ItKubernetesNameSpaceWatchingEvents, **/ItMiiAuxiliaryImage, **/ItMiiAuxiliaryImageCluster, **/ItMiiDomain, @@ -500,7 +501,8 @@ false **/ItCrossDomainTransaction, - **/ItKubernetesEvents, + **/ItKubernetesDomainEvents, + **/ItKubernetesNameSpaceWatchingEvents, **/ItMiiAuxiliaryImage, **/ItMiiAuxiliaryImageCluster, **/ItMiiDomain, diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesEvents.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesDomainEvents.java similarity index 61% rename from integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesEvents.java rename to integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesDomainEvents.java index b5fe5cf8c89..72ae921dddf 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesEvents.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesDomainEvents.java @@ -1,4 +1,4 @@ -// Copyright (c) 2020, 2022, Oracle and/or its affiliates. +// Copyright (c) 2022, Oracle and/or its affiliates. // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. package oracle.weblogic.kubernetes; @@ -15,8 +15,6 @@ import java.util.Properties; import io.kubernetes.client.custom.V1Patch; -import io.kubernetes.client.openapi.ApiException; -import io.kubernetes.client.openapi.models.CoreV1Event; import io.kubernetes.client.openapi.models.V1Container; import io.kubernetes.client.openapi.models.V1EnvVar; import io.kubernetes.client.openapi.models.V1LocalObjectReference; @@ -25,7 +23,6 @@ import io.kubernetes.client.openapi.models.V1SecretReference; import io.kubernetes.client.openapi.models.V1Volume; import io.kubernetes.client.openapi.models.V1VolumeMount; -import io.kubernetes.client.util.Yaml; import oracle.weblogic.domain.AdminServer; import oracle.weblogic.domain.AdminService; import oracle.weblogic.domain.Channel; @@ -34,8 +31,6 @@ import oracle.weblogic.domain.DomainSpec; import oracle.weblogic.domain.ServerPod; import oracle.weblogic.kubernetes.actions.impl.OperatorParams; -import oracle.weblogic.kubernetes.actions.impl.primitive.Command; -import oracle.weblogic.kubernetes.actions.impl.primitive.CommandParams; import oracle.weblogic.kubernetes.annotations.IntegrationTest; import oracle.weblogic.kubernetes.annotations.Namespaces; import oracle.weblogic.kubernetes.logging.LoggingFacade; @@ -43,13 +38,8 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; @@ -61,7 +51,6 @@ import static oracle.weblogic.kubernetes.TestConstants.SKIP_CLEANUP; import static oracle.weblogic.kubernetes.TestConstants.WEBLOGIC_IMAGE_TO_USE_IN_SPEC; import static oracle.weblogic.kubernetes.actions.ActionConstants.RESOURCE_DIR; -import static oracle.weblogic.kubernetes.actions.TestActions.createNamespace; import static oracle.weblogic.kubernetes.actions.TestActions.deleteDomainCustomResource; import static oracle.weblogic.kubernetes.actions.TestActions.deletePersistentVolume; import static oracle.weblogic.kubernetes.actions.TestActions.deletePersistentVolumeClaim; @@ -71,14 +60,19 @@ import static oracle.weblogic.kubernetes.actions.TestActions.getServicePort; import static oracle.weblogic.kubernetes.actions.TestActions.now; import static oracle.weblogic.kubernetes.actions.TestActions.scaleClusterWithRestApi; +import static oracle.weblogic.kubernetes.actions.TestActions.shutdownDomain; import static oracle.weblogic.kubernetes.actions.impl.Domain.patchDomainCustomResource; import static oracle.weblogic.kubernetes.assertions.TestAssertions.verifyRollingRestartOccurred; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkServiceExists; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getNextFreePort; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getUniqueName; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntil; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.withLongRetryPolicy; import static oracle.weblogic.kubernetes.utils.ConfigMapUtils.createConfigMapForDomainCreation; import static oracle.weblogic.kubernetes.utils.DomainUtils.createDomainAndVerify; +import static oracle.weblogic.kubernetes.utils.DomainUtils.getAndValidateInitialDomain; +import static oracle.weblogic.kubernetes.utils.DomainUtils.verifyDomainStatusConditionTypeDoesNotExist; import static oracle.weblogic.kubernetes.utils.ImageUtils.createSecretForBaseImages; import static oracle.weblogic.kubernetes.utils.JobUtils.createDomainJob; import static oracle.weblogic.kubernetes.utils.JobUtils.getIntrospectJobName; @@ -93,22 +87,21 @@ import static oracle.weblogic.kubernetes.utils.K8sEvents.DOMAIN_ROLL_COMPLETED; import static oracle.weblogic.kubernetes.utils.K8sEvents.DOMAIN_ROLL_STARTING; import static oracle.weblogic.kubernetes.utils.K8sEvents.DOMAIN_VALIDATION_ERROR; -import static oracle.weblogic.kubernetes.utils.K8sEvents.NAMESPACE_WATCHING_STARTED; import static oracle.weblogic.kubernetes.utils.K8sEvents.POD_CYCLE_STARTING; import static oracle.weblogic.kubernetes.utils.K8sEvents.checkDomainEvent; import static oracle.weblogic.kubernetes.utils.K8sEvents.checkDomainEventWatchingStopped; import static oracle.weblogic.kubernetes.utils.K8sEvents.checkDomainEventWithCount; +import static oracle.weblogic.kubernetes.utils.K8sEvents.checkDomainFailedEventWithReason; import static oracle.weblogic.kubernetes.utils.K8sEvents.domainEventExists; import static oracle.weblogic.kubernetes.utils.K8sEvents.getDomainEventCount; -import static oracle.weblogic.kubernetes.utils.K8sEvents.getEvent; import static oracle.weblogic.kubernetes.utils.K8sEvents.getEventCount; import static oracle.weblogic.kubernetes.utils.OKDUtils.createRouteForOKD; import static oracle.weblogic.kubernetes.utils.OKDUtils.setTlsTerminationForRoute; import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator; -import static oracle.weblogic.kubernetes.utils.OperatorUtils.upgradeAndVerifyOperator; import static oracle.weblogic.kubernetes.utils.PatchDomainUtils.patchDomainResource; import static oracle.weblogic.kubernetes.utils.PersistentVolumeUtils.createPV; import static oracle.weblogic.kubernetes.utils.PersistentVolumeUtils.createPVC; +import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodDeleted; import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodDoesNotExist; import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodExists; import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodReady; @@ -122,6 +115,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -134,10 +128,10 @@ * The tests creates the domain resource, modifies it, introduces some validation errors in the domain resource * and finally deletes it to generate all the domain related events. */ -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) + @DisplayName("Verify the Kubernetes events for domain lifecycle") @IntegrationTest -class ItKubernetesEvents { +class ItKubernetesDomainEvents { private static String opNamespace = null; private static String domainNamespace1 = null; @@ -149,19 +143,34 @@ class ItKubernetesEvents { private static int externalRestHttpsPort = 0; private static OperatorParams opParams = null; - final String cluster1Name = "mycluster"; - final String cluster2Name = "cl2"; - final String adminServerName = "admin-server"; - final String domainUid = "k8seventsdomain"; - final String adminServerPodName = domainUid + "-" + adminServerName; - final String managedServerNameBase = "ms-"; - String managedServerPodNamePrefix = domainUid + "-" + managedServerNameBase; - final int managedServerPort = 8001; - int replicaCount = 2; - - final String pvName = getUniqueName(domainUid + "-pv-"); - final String pvcName = getUniqueName(domainUid + "-pvc-"); - private final String wlSecretName = "weblogic-credentials"; + static final String cluster1Name = "mycluster"; + static final String cluster2Name = "cl2"; + static final String adminServerName = "admin-server"; + static final String domainUid = "k8seventsdomain"; + static final String adminServerPodName = domainUid + "-" + adminServerName; + static final String managedServerNameBase = "ms-"; + static String managedServerPodNamePrefix = domainUid + "-" + managedServerNameBase; + static final int managedServerPort = 8001; + static int replicaCount = 2; + + static final String pvName1 = getUniqueName(domainUid + "-pv-"); + static final String pvcName1 = getUniqueName(domainUid + "-pvc-"); + static final String pvName2 = getUniqueName(domainUid + "-pv-"); + static final String pvcName2 = getUniqueName(domainUid + "-pvc-"); + static final String pvName3 = getUniqueName(domainUid + "-pv-"); + static final String pvcName3 = getUniqueName(domainUid + "-pvc-"); + static final String pvName4 = getUniqueName(domainUid + "-pv-"); + static final String pvcName4 = getUniqueName(domainUid + "-pvc-"); + static final String pvName5 = getUniqueName(domainUid + "-pv-"); + static final String pvcName5 = getUniqueName(domainUid + "-pvc-"); + private static final String wlSecretName = "weblogic-credentials"; + private static String className = new Object(){}.getClass().getEnclosingClass().getSimpleName(); + + public enum ScaleAction { + scaleUp, + scaleDown, + reset + } // create standard, reusable retry/backoff policy private static final ConditionFactory withStandardRetryPolicy @@ -196,11 +205,14 @@ public static void initAll(@Namespaces(6) List namespaces) { assertNotNull(namespaces.get(5), "Namespace is null"); domainNamespace5 = namespaces.get(5); + // set the service account name for the operator opServiceAccount = opNamespace + "-sa"; // install and verify operator with REST API - opParams = installAndVerifyOperator(opNamespace, opServiceAccount, true, 0, domainNamespace1); + opParams = installAndVerifyOperator(opNamespace, opServiceAccount, + true, 0, domainNamespace1, domainNamespace2, domainNamespace3, + domainNamespace4, domainNamespace5); externalRestHttpsPort = getServiceNodePort(opNamespace, "external-weblogic-operator-svc"); // This test uses the operator restAPI to scale the domain. To do this in OKD cluster, @@ -210,9 +222,7 @@ public static void initAll(@Namespaces(6) List namespaces) { // Patch the route just created to set tls termination to passthrough setTlsTerminationForRoute("external-weblogic-operator-svc", opNamespace); - // create pull secrets for WebLogic image when running in non Kind Kubernetes cluster - // this secret is used only for non-kind cluster - createSecretForBaseImages(domainNamespace1); + createDomain(domainNamespace3, domainUid, pvName3, pvcName3); } /** @@ -221,27 +231,29 @@ public static void initAll(@Namespaces(6) List namespaces) { * Verifies DomainProcessingStarting is logged when operator processes the domain resource. * Verifies DomainProcessingCompleted is logged when operator done processes the domain resource. */ - @Order(1) @Test @DisplayName("Test domain events for various successful domain life cycle changes") @Tag("gate") void testDomainK8SEventsSuccess() { - OffsetDateTime timestamp = now(); - logger.info("Creating domain"); - createDomain(); - - logger.info("verify the DomainCreated event is generated"); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_CREATED, "Normal", timestamp); - logger.info("verify the DomainProcessing Starting/Completed event is generated"); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_PROCESSING_STARTING, "Normal", timestamp); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_PROCESSING_COMPLETED, "Normal", timestamp); + try { + OffsetDateTime timestamp = now(); + logger.info("Creating domain"); + createDomain(domainNamespace1, domainUid, pvName1, pvcName1); + + logger.info("verify the DomainCreated event is generated"); + checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_CREATED, "Normal", timestamp); + logger.info("verify the DomainProcessing Starting/Completed event is generated"); + checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_PROCESSING_STARTING, "Normal", timestamp); + checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_PROCESSING_COMPLETED, "Normal", timestamp); + } finally { + shutdownDomain(domainUid, domainNamespace1); + } } /** * Patch a domain resource with a new managed server not existing in actual WebLogic domain and verify * the warning DomainValidationError event is logged by the operator in the domain namespace. */ - @Order(2) @Test @DisplayName("Test domain DomainValidationError event for non-existing managed server") void testDomainK8sEventsNonExistingManagedServer() { @@ -256,10 +268,10 @@ void testDomainK8sEventsNonExistingManagedServer() { + "}]"; logger.info("Updating domain configuration using patch string: {0}\n", patchStr); V1Patch patch = new V1Patch(patchStr); - assertTrue(patchDomainCustomResource(domainUid, domainNamespace1, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), + assertTrue(patchDomainCustomResource(domainUid, domainNamespace3, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), "Failed to patch domain"); logger.info("verify the DomainValidationError event is generated"); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_VALIDATION_ERROR, "Warning", timestamp); + checkEvent(opNamespace, domainNamespace3, domainUid, DOMAIN_VALIDATION_ERROR, "Warning", timestamp); // remove the managed server from domain resource timestamp = now(); @@ -269,18 +281,17 @@ void testDomainK8sEventsNonExistingManagedServer() { + "}]"; logger.info("Updating domain configuration using patch string: {0}\n", patchStr); patch = new V1Patch(patchStr); - assertTrue(patchDomainCustomResource(domainUid, domainNamespace1, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), + assertTrue(patchDomainCustomResource(domainUid, domainNamespace3, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), "Failed to patch domain"); logger.info("verify the DomainChanged event is generated"); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_CHANGED, "Normal", timestamp); + checkEvent(opNamespace, domainNamespace3, domainUid, DOMAIN_CHANGED, "Normal", timestamp); } /** * Patch a domain resource with a new cluster not existing in actual WebLogic domain and verify * the warning DomainValidationError event is logged by the operator in the domain namespace. */ - @Order(3) @Test @DisplayName("Test domain DomainValidationError event for non-existing cluster") void testDomainK8sEventsNonExistingCluster() { @@ -293,21 +304,21 @@ void testDomainK8sEventsNonExistingCluster() { + "}]"; logger.info("Updating domain configuration using patch string: {0}\n", patchStr); V1Patch patch = new V1Patch(patchStr); - assertTrue(patchDomainCustomResource(domainUid, domainNamespace1, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), + assertTrue(patchDomainCustomResource(domainUid, domainNamespace3, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), "Failed to patch domain"); // verify the DomainValidationError event is generated - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_VALIDATION_ERROR, "Warning", timestamp); + checkEvent(opNamespace, domainNamespace3, domainUid, DOMAIN_VALIDATION_ERROR, "Warning", timestamp); //remove the cluster from domain resource timestamp = now(); patchStr = "[{\"op\": \"remove\",\"path\": \"/spec/clusters/1\"}]"; logger.info("Updating domain configuration using patch string: {0}\n", patchStr); patch = new V1Patch(patchStr); - assertTrue(patchDomainCustomResource(domainUid, domainNamespace1, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), + assertTrue(patchDomainCustomResource(domainUid, domainNamespace3, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), "Failed to patch domain"); // verify the DomainProcessingStarting/Completed event is generated - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_CHANGED, "Normal", timestamp); + checkEvent(opNamespace, domainNamespace3, domainUid, DOMAIN_CHANGED, "Normal", timestamp); } /** @@ -319,76 +330,47 @@ void testDomainK8sEventsNonExistingCluster() { * up processing the domain resource. * Cleanup by patching the domain resource to a valid location and introspectVersion to bring up all servers again. */ - @Order(4) @Test @DisplayName("Test domain events for failed/retried domain life cycle changes") - void testDomainK8SEventsFailed() { - V1Patch patch; - String patchStr; - Domain domain = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, domainNamespace1)); - String originalDomainHome = domain.getSpec().getDomainHome(); - - OffsetDateTime timestamp = now(); + void testK8SEventsFailedLifeCycle() { try { - logger.info("Shutting down all servers in domain with serverStartPolicy : NEVER"); - patchStr = "[{\"op\": \"replace\", \"path\": \"/spec/serverStartPolicy\", \"value\": \"NEVER\"}]"; - patch = new V1Patch(patchStr); - assertTrue(patchDomainCustomResource(domainUid, domainNamespace1, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), - "patchDomainCustomResource failed"); + V1Patch patch; + String patchStr; + Domain domain = createDomain(domainNamespace5, domainUid, pvName5, pvcName5, "NEVER"); + assertNotNull(domain, " Can't create domain resource"); + + String originalDomainHome = domain.getSpec().getDomainHome(); + + OffsetDateTime timestamp = now(); logger.info("Checking if the admin server {0} is shutdown in namespace {1}", - adminServerPodName, domainNamespace1); - checkPodDoesNotExist(adminServerPodName, domainUid, domainNamespace1); + adminServerPodName, domainNamespace5); + checkPodDoesNotExist(adminServerPodName, domainUid, domainNamespace5); for (int i = 1; i <= replicaCount; i++) { logger.info("Checking if the managed server {0} is shutdown in namespace {1}", - managedServerPodNamePrefix + i, domainNamespace1); - checkPodDoesNotExist(managedServerPodNamePrefix + i, domainUid, domainNamespace1); + managedServerPodNamePrefix + i, domainNamespace5); + checkPodDoesNotExist(managedServerPodNamePrefix + i, domainUid, domainNamespace5); } logger.info("Replace the domainHome to a nonexisting location to verify the following events" - + " DomainChanged, DomainProcessingRetrying and DomainProcessingAborted are logged"); + + " Changed and Failed events are logged"); patchStr = "[{\"op\": \"replace\", " + "\"path\": \"/spec/domainHome\", \"value\": \"" + originalDomainHome + "bad\"}," + "{\"op\": \"replace\", \"path\": \"/spec/serverStartPolicy\", \"value\": \"IF_NEEDED\"}]"; logger.info("PatchStr for domainHome: {0}", patchStr); patch = new V1Patch(patchStr); - assertTrue(patchDomainCustomResource(domainUid, domainNamespace1, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), + assertTrue(patchDomainCustomResource(domainUid, domainNamespace5, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), "patchDomainCustomResource failed"); logger.info("verify domain changed event is logged"); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_CHANGED, "Normal", timestamp); - logger.info("verify domain processing retrying event"); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_PROCESSING_RETRYING, "Normal", timestamp); - logger.info("verify domain processing aborted event"); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_PROCESSING_ABORTED, "Warning", timestamp); + checkEvent(opNamespace, domainNamespace5, domainUid, DOMAIN_CHANGED, "Normal", timestamp); + checkEvent(opNamespace, domainNamespace5, domainUid, DOMAIN_PROCESSING_RETRYING, "Normal", timestamp); + logger.info("verify domain failed event"); + checkEvent(opNamespace, domainNamespace5, domainUid, DOMAIN_PROCESSING_ABORTED, "Warning", timestamp); } finally { - logger.info("Restoring the domain with valid location and bringing up all servers"); - timestamp = now(); - String introspectVersion = assertDoesNotThrow(() -> getNextIntrospectVersion(domainUid, domainNamespace1)); - // add back the original domain home - patchStr = "[" - + "{\"op\": \"replace\", \"path\": \"/spec/domainHome\", \"value\": \"" + originalDomainHome + "\"}," - + "{\"op\": \"add\", \"path\": \"/spec/introspectVersion\", \"value\": \"" + introspectVersion + "\"}" - + "]"; - logger.info("PatchStr for domainHome: {0}", patchStr); - - patch = new V1Patch(patchStr); - assertTrue(patchDomainCustomResource(domainUid, domainNamespace1, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), - "patchDomainCustomResource failed"); - - logger.info("verify domain changed event is logged"); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_CHANGED, "Normal", timestamp); - logger.info("verifying the admin server is created and started"); - checkPodReadyAndServiceExists(adminServerPodName, domainUid, domainNamespace1); - - // verify managed server services created - for (int i = 1; i <= replicaCount; i++) { - logger.info("Checking managed server service/pod {0} is created in namespace {1}", - managedServerPodNamePrefix + i, domainNamespace1); - checkPodReadyAndServiceExists(managedServerPodNamePrefix + i, domainUid, domainNamespace1); - } + shutdownDomain(domainUid, domainNamespace5); } } @@ -399,26 +381,24 @@ void testDomainK8SEventsFailed() { * and starts up the new cluster. * Verifies the scaling operation generates only one DomainProcessing Starting/Completed. */ - @Order(5) @Test void testK8SEventsMultiClusterEvents() { - createNewCluster(); OffsetDateTime timestamp = now(); + createNewCluster(domainNamespace3); scaleClusterWithRestApi(domainUid, cluster2Name, 1, externalRestHttpsPort, opNamespace, opServiceAccount); logger.info("verify the DomainProcessing Starting/Completed event is generated"); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_PROCESSING_STARTING, "Normal", timestamp); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_PROCESSING_COMPLETED, "Normal", timestamp); + checkEvent(opNamespace, domainNamespace3, domainUid, DOMAIN_PROCESSING_STARTING, "Normal", timestamp); + checkEvent(opNamespace, domainNamespace3, domainUid, DOMAIN_PROCESSING_COMPLETED, "Normal", timestamp); logger.info("verify the only 1 DomainProcessing Starting/Completed event is generated"); - assertEquals(1, getEventCount(domainNamespace1, domainUid, DOMAIN_PROCESSING_STARTING, timestamp)); - assertEquals(1, getEventCount(domainNamespace1, domainUid, DOMAIN_PROCESSING_COMPLETED, timestamp)); + assertEquals(1, getEventCount(domainNamespace3, domainUid, DOMAIN_PROCESSING_STARTING, timestamp)); + assertEquals(1, getEventCount(domainNamespace3, domainUid, DOMAIN_PROCESSING_COMPLETED, timestamp)); } /** * Scale the cluster beyond maximum dynamic cluster size and verify the * DomainValidationError warning event is generated. */ - @Order(6) @Test void testDomainK8sEventsScalePastMax() { OffsetDateTime timestamp = now(); @@ -430,13 +410,12 @@ void testDomainK8sEventsScalePastMax() { + "]"; logger.info("Updating replicas in cluster {0} using patch string: {1}", cluster1Name, patchStr); V1Patch patch = new V1Patch(patchStr); - assertTrue(patchDomainCustomResource(domainUid, domainNamespace1, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), + assertTrue(patchDomainCustomResource(domainUid, domainNamespace3, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), "Failed to patch domain"); - logger.info("verify the DomainValidationError event is generated"); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_VALIDATION_ERROR, "Warning", timestamp); + logger.info("verify the Failed event is generated"); + checkEvent(opNamespace, domainNamespace3, domainUid, DOMAIN_VALIDATION_ERROR, "Warning", timestamp); } finally { - timestamp = now(); logger.info("Updating domain resource to set correct replicas size"); String patchStr = "[" @@ -444,7 +423,7 @@ void testDomainK8sEventsScalePastMax() { + "]"; logger.info("Updating replicas in cluster {0} using patch string: {1}", cluster1Name, patchStr); V1Patch patch = new V1Patch(patchStr); - assertTrue(patchDomainCustomResource(domainUid, domainNamespace1, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), + assertTrue(patchDomainCustomResource(domainUid, domainNamespace3, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), "Failed to patch domain"); } } @@ -453,15 +432,15 @@ void testDomainK8sEventsScalePastMax() { * Scale down and scale up the domain and verify that * DomainProcessingCompleted normal event is generated. */ - @Order(7) @Test @DisplayName("Test domain completed event when domain is scaled.") void testScaleDomainAndVerifyCompletedEvent() { try { - scaleDomainAndVerifyCompletedEvent(1, "scale down", true); - scaleDomainAndVerifyCompletedEvent(2, "scale up", true); + createDomain(domainNamespace4, domainUid, pvName4, pvcName4); + scaleDomainAndVerifyCompletedEvent(1, ScaleAction.scaleDown, true, domainNamespace4); + scaleDomainAndVerifyCompletedEvent(2, ScaleAction.scaleUp, true, domainNamespace4); } finally { - scaleDomain(2); + shutdownDomain(domainUid, domainNamespace4); } } @@ -469,7 +448,6 @@ void testScaleDomainAndVerifyCompletedEvent() { * Scale the cluster below minimum dynamic cluster size and verify the DomainValidationError * warning event is generated. */ - @Order(8) @Test void testDomainK8sEventsScaleBelowMin() { OffsetDateTime timestamp = now(); @@ -481,11 +459,13 @@ void testDomainK8sEventsScaleBelowMin() { + "]"; logger.info("Updating replicas in cluster {0} using patch string: {1}", cluster1Name, patchStr); V1Patch patch = new V1Patch(patchStr); - assertTrue(patchDomainCustomResource(domainUid, domainNamespace1, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), + assertTrue(patchDomainCustomResource(domainUid, domainNamespace3, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), "Failed to patch domain"); - logger.info("verify the DomainValidationError event is generated"); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_VALIDATION_ERROR, "Warning", timestamp); + // No event will be created for this + logger.info("verify the Failed event is NOT generated"); + assertFalse(domainEventExists(opNamespace, domainNamespace3, domainUid, + DOMAIN_VALIDATION_ERROR, "Warning", timestamp)); } finally { timestamp = now(); logger.info("Updating domain resource to set correct replicas size"); @@ -495,7 +475,7 @@ void testDomainK8sEventsScaleBelowMin() { + "]"; logger.info("Updating replicas in cluster {0} using patch string: {1}", cluster1Name, patchStr); V1Patch patch = new V1Patch(patchStr); - assertTrue(patchDomainCustomResource(domainUid, domainNamespace1, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), + assertTrue(patchDomainCustomResource(domainUid, domainNamespace3, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), "Failed to patch domain"); } } @@ -504,7 +484,6 @@ void testDomainK8sEventsScaleBelowMin() { * Replace the pv and pvc in the domain resource with a pv/pvc not containing any WebLogic domain * and verify the DomainProcessingFailed warning event is generated. */ - @Order(9) @Test void testDomainK8sEventsProcessingFailed() { OffsetDateTime timestamp = now(); @@ -512,8 +491,8 @@ void testDomainK8sEventsProcessingFailed() { String pvName = getUniqueName("sample-pv-"); String pvcName = getUniqueName("sample-pvc-"); createPV(pvName, domainUid, this.getClass().getSimpleName()); - createPVC(pvName, pvcName, domainUid, domainNamespace1); - String introspectVersion = assertDoesNotThrow(() -> getNextIntrospectVersion(domainUid, domainNamespace1)); + createPVC(pvName, pvcName, domainUid, domainNamespace3); + String introspectVersion = assertDoesNotThrow(() -> getNextIntrospectVersion(domainUid, domainNamespace3)); String patchStr = "[" + "{\"op\": \"replace\", \"path\": \"/spec/serverPod/volumeMounts/0/name\", \"value\": \"sample-pv\"}," @@ -524,37 +503,36 @@ void testDomainK8sEventsProcessingFailed() { + "]"; logger.info("Updating pv/pvcs in domain resource using patch string: {0}", patchStr); V1Patch patch = new V1Patch(patchStr); - assertTrue(patchDomainCustomResource(domainUid, domainNamespace1, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), + assertTrue(patchDomainCustomResource(domainUid, domainNamespace3, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), "Failed to patch domain"); logger.info("verify the DomainProcessingFailed event is generated"); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); + checkEvent(opNamespace, domainNamespace3, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); } finally { - String introspectVersion = assertDoesNotThrow(() -> getNextIntrospectVersion(domainUid, domainNamespace1)); + String introspectVersion = assertDoesNotThrow(() -> getNextIntrospectVersion(domainUid, domainNamespace3)); String patchStr = "[" - + "{\"op\": \"replace\", \"path\": \"/spec/serverPod/volumeMounts/0/name\", \"value\": \"" + pvName + "\"}," - + "{\"op\": \"replace\", \"path\": \"/spec/serverPod/volumes/0/name\", \"value\": \"" + pvName + "\"}," + + "{\"op\": \"replace\", \"path\": \"/spec/serverPod/volumeMounts/0/name\", \"value\": \"" + pvName3 + "\"}," + + "{\"op\": \"replace\", \"path\": \"/spec/serverPod/volumes/0/name\", \"value\": \"" + pvName3 + "\"}," + "{\"op\": \"replace\", \"path\": " - + "\"/spec/serverPod/volumes/0/persistentVolumeClaim/claimName\", \"value\": \"" + pvcName + "\"}," + + "\"/spec/serverPod/volumes/0/persistentVolumeClaim/claimName\", \"value\": \"" + pvcName3 + "\"}," + "{\"op\": \"add\", \"path\": \"/spec/introspectVersion\", \"value\": \"" + introspectVersion + "\"}" + "]"; logger.info("Updating pv/pvcs in domain resource using patch string: {0}", patchStr); V1Patch patch = new V1Patch(patchStr); timestamp = now(); - assertTrue(patchDomainCustomResource(domainUid, domainNamespace1, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), + assertTrue(patchDomainCustomResource(domainUid, domainNamespace3, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), "Failed to patch domain"); logger.info("verify domain changed/processing completed events are logged"); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_CHANGED, "Normal", timestamp); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_PROCESSING_COMPLETED, "Normal", timestamp); + checkEvent(opNamespace, domainNamespace3, domainUid, DOMAIN_PROCESSING_STARTING, "Normal", timestamp); + checkEvent(opNamespace, domainNamespace3, domainUid, DOMAIN_PROCESSING_COMPLETED, "Normal", timestamp); } } /** * The test modifies the logHome property and verifies the domain roll events are logged. */ - @Order(10) @Test @DisplayName("Verify logHome property change rolls domain and relevant events are logged") void testLogHomeChangeEvents() { @@ -562,12 +540,10 @@ void testLogHomeChangeEvents() { OffsetDateTime timestamp = now(); // get the original domain resource before update - Domain domain1 = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, domainNamespace1), - String.format("getDomainCustomResource failed with ApiException when tried to get domain %s in namespace %s", - domainUid, domainNamespace1)); + Domain domain1 = getAndValidateInitialDomain(domainNamespace3, domainUid); // get the map with server pods and their original creation timestamps - Map podsWithTimeStamps = getPodsWithTimeStamps(domainNamespace1, + Map podsWithTimeStamps = getPodsWithTimeStamps(domainNamespace3, adminServerPodName, managedServerPodNamePrefix, replicaCount); //print out the original image name @@ -580,56 +556,40 @@ void testLogHomeChangeEvents() { + "]"; logger.info("PatchStr for logHome: {0}", patchStr); - assertTrue(patchDomainResource(domainUid, domainNamespace1, new StringBuffer(patchStr)), + assertTrue(patchDomainResource(domainUid, domainNamespace3, new StringBuffer(patchStr)), "patchDomainCustomResource(logHome) failed"); - domain1 = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, domainNamespace1), - String.format("getDomainCustomResource failed with ApiException when tried to get domain %s in namespace %s", - domainUid, domainNamespace1)); + domain1 = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, domainNamespace3), + String.format("getDomainCustomResource failed with ApiException" + + " when tried to get domain %s in namespace %s", + domainUid, domainNamespace3)); //print out logHome in the new patched domain logger.info("In the new patched domain logHome is: {0}", domain1.getSpec().getLogHome()); - assertTrue(domain1.getSpec().getLogHome().equals("/shared/logs/logHome"), "logHome is not updated"); + assertEquals("/shared/logs/logHome", domain1.getSpec().getLogHome(), "logHome is not updated"); // verify the server pods are rolling restarted and back to ready state logger.info("Verifying rolling restart occurred for domain {0} in namespace {1}", - domainUid, domainNamespace1); - assertTrue(verifyRollingRestartOccurred(podsWithTimeStamps, 1, domainNamespace1), - String.format("Rolling restart failed for domain %s in namespace %s", domainUid, domainNamespace1)); + domainUid, domainNamespace3); + assertTrue(verifyRollingRestartOccurred(podsWithTimeStamps, 1, domainNamespace3), + String.format("Rolling restart failed for domain %s in namespace %s", domainUid, domainNamespace3)); - checkPodReadyAndServiceExists(adminServerPodName, domainUid, domainNamespace1); + checkPodReadyAndServiceExists(adminServerPodName, domainUid, domainNamespace3); for (int i = 1; i <= replicaCount; i++) { logger.info("Checking managed server service {0} is created in namespace {1}", - managedServerPodNamePrefix + i, domainNamespace1); - checkPodReadyAndServiceExists(managedServerPodNamePrefix + i, domainUid, domainNamespace1); + managedServerPodNamePrefix + i, domainNamespace3); + checkPodReadyAndServiceExists(managedServerPodNamePrefix + i, domainUid, domainNamespace3); } //verify the logHome change causes the domain roll events to be logged - logger.info("verify domain roll starting/pod cycle starting events are logged"); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_ROLL_STARTING, "Normal", timestamp); - checkEvent(opNamespace, domainNamespace1, domainUid, POD_CYCLE_STARTING, "Normal", timestamp); - - CoreV1Event event = getEvent(opNamespace, domainNamespace1, - domainUid, DOMAIN_ROLL_STARTING, "Normal", timestamp); - logger.info(Yaml.dump(event)); - logger.info("verify the event message contains the logHome changed messages is logged"); - assertTrue(event.getMessage().contains("logHome")); - - event = getEvent(opNamespace, domainNamespace1, - domainUid, POD_CYCLE_STARTING, "Normal", timestamp); - logger.info(Yaml.dump(event)); - logger.info("verify the event message contains the LOG_HOME changed messages is logged"); - assertTrue(event.getMessage().contains("LOG_HOME")); - - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_ROLL_COMPLETED, "Normal", timestamp); + verifyDomainRollAndPodCycleEvents(timestamp, domainNamespace3); } /** * The test modifies the includeServerOutInPodLog property and verifies the domain roll starting events are logged. */ - @Order(11) @Test @DisplayName("Verify includeServerOutInPodLog property change rolls domain and relevant events are logged") void testIncludeServerOutInPodLog() { @@ -637,12 +597,10 @@ void testIncludeServerOutInPodLog() { OffsetDateTime timestamp = now(); // get the original domain resource before update - Domain domain1 = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, domainNamespace1), - String.format("getDomainCustomResource failed with ApiException when tried to get domain %s in namespace %s", - domainUid, domainNamespace1)); + Domain domain1 = getAndValidateInitialDomain(domainNamespace3, domainUid); // get the map with server pods and their original creation timestamps - Map podsWithTimeStamps = getPodsWithTimeStamps(domainNamespace1, + Map podsWithTimeStamps = getPodsWithTimeStamps(domainNamespace3, adminServerPodName, managedServerPodNamePrefix, replicaCount); //print out the original includeServerOutInPodLog value @@ -656,298 +614,53 @@ void testIncludeServerOutInPodLog() { + "]"; logger.info("PatchStr for includeServerOutInPodLog: {0}", patchStr); - assertTrue(patchDomainResource(domainUid, domainNamespace1, new StringBuffer(patchStr)), + assertTrue(patchDomainResource(domainUid, domainNamespace3, new StringBuffer(patchStr)), "patchDomainCustomResource(includeServerOutInPodLog) failed"); - domain1 = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, domainNamespace1), - String.format("getDomainCustomResource failed with ApiException when tried to get domain %s in namespace %s", - domainUid, domainNamespace1)); + domain1 = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, domainNamespace3), + String.format("getDomainCustomResource failed with " + + "ApiException when tried to get domain %s in namespace %s", + domainUid, domainNamespace3)); //print out includeServerOutInPodLog in the new patched domain logger.info("In the new patched domain includeServerOutInPodLog is: {0}", domain1.getSpec().includeServerOutInPodLog()); - assertTrue(domain1.getSpec().includeServerOutInPodLog() != includeLogInPod, + assertNotEquals(includeLogInPod, domain1.getSpec().includeServerOutInPodLog(), "includeServerOutInPodLog is not updated"); // verify the server pods are rolling restarted and back to ready state logger.info("Verifying rolling restart occurred for domain {0} in namespace {1}", - domainUid, domainNamespace1); - assertTrue(verifyRollingRestartOccurred(podsWithTimeStamps, 1, domainNamespace1), - String.format("Rolling restart failed for domain %s in namespace %s", domainUid, domainNamespace1)); + domainUid, domainNamespace3); + assertTrue(verifyRollingRestartOccurred(podsWithTimeStamps, 1, domainNamespace3), + String.format("Rolling restart failed for domain %s in namespace %s", domainUid, domainNamespace3)); - checkPodReadyAndServiceExists(adminServerPodName, domainUid, domainNamespace1); + checkPodReadyAndServiceExists(adminServerPodName, domainUid, domainNamespace3); for (int i = 1; i <= replicaCount; i++) { logger.info("Checking managed server service {0} is created in namespace {1}", - managedServerPodNamePrefix + i, domainNamespace1); - checkPodReadyAndServiceExists(managedServerPodNamePrefix + i, domainUid, domainNamespace1); + managedServerPodNamePrefix + i, domainNamespace3); + checkPodReadyAndServiceExists(managedServerPodNamePrefix + i, domainUid, domainNamespace3); } //verify the includeServerOutInPodLog change causes the domain roll events to be logged - logger.info("verify domain roll starting/pod cycle starting events are logged"); - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_ROLL_STARTING, "Normal", timestamp); - checkEvent(opNamespace, domainNamespace1, domainUid, POD_CYCLE_STARTING, "Normal", timestamp); - - CoreV1Event event = getEvent(opNamespace, domainNamespace1, - domainUid, DOMAIN_ROLL_STARTING, "Normal", timestamp); - logger.info(Yaml.dump(event)); - logger.info("verify the event message contains the includeServerOutInPodLog changed messages is logged"); - assertTrue(event.getMessage().contains("isIncludeServerOutInPodLog")); - - event = getEvent(opNamespace, domainNamespace1, domainUid, POD_CYCLE_STARTING, "Normal", timestamp); - logger.info(Yaml.dump(event)); - logger.info("verify the event message contains the SERVER_OUT_IN_POD_LOG changed messages is logged"); - assertTrue(event.getMessage().contains("SERVER_OUT_IN_POD_LOG")); - - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_ROLL_COMPLETED, "Normal", timestamp); + verifyDomainRollAndPodCycleEvents(timestamp, domainNamespace3); } /** * Test DomainDeleted event is logged when domain resource is deleted. */ - @Order(13) @Test @DisplayName("Test domain events for various domain life cycle changes") void testDomainK8SEventsDelete() { OffsetDateTime timestamp = now(); - - deleteDomainCustomResource(domainUid, domainNamespace1); - checkPodDoesNotExist(adminServerPodName, domainUid, domainNamespace1); - checkPodDoesNotExist(managedServerPodNamePrefix + 1, domainUid, domainNamespace1); - checkPodDoesNotExist(managedServerPodNamePrefix + 2, domainUid, domainNamespace1); + createDomain(domainNamespace2, domainUid, pvName2, pvcName2); + deleteDomainCustomResource(domainUid, domainNamespace2); + checkPodDoesNotExist(adminServerPodName, domainUid, domainNamespace2); + checkPodDoesNotExist(managedServerPodNamePrefix + 1, domainUid, domainNamespace2); + checkPodDoesNotExist(managedServerPodNamePrefix + 2, domainUid, domainNamespace2); //verify domain deleted event - checkEvent(opNamespace, domainNamespace1, domainUid, DOMAIN_DELETED, "Normal", timestamp); - } - - /** - * Test verifies the operator logs a NamespaceWatchingStarted event in the respective domain namespace - * when it starts watching a new domain namespace with domainNamespaceSelectionStrategy default to List and - * operator logs a NamespaceWatchingStopped event in the respective domain namespace - * when it stops watching a domain namespace. - * The test upgrades the operator instance through helm to add or remove another domain namespace - * in the operator watch list. - * This is a parameterized test with enableClusterRoleBinding set to either true or false. - *

- *

{@literal
-   * helm upgrade weblogic-operator kubernetes/charts/weblogic-operator
-   * --namespace ns-ipqy
-   * --reuse-values
-   * --set "domainNamespaces={ns-xghr,ns-idir}"
-   * }
-   * 
- *

- */ - @Order(14) - @ParameterizedTest - @ValueSource(booleans = { true, false }) - void testK8SEventsStartStopWatchingNS(boolean enableClusterRoleBinding) { - logger.info("testing testK8SEventsStartStopWatchingNS with enableClusterRoleBinding={0}", - enableClusterRoleBinding); - OffsetDateTime timestamp = now(); - - logger.info("Adding a new domain namespace in the operator watch list"); - List domainNamespaces = new ArrayList<>(); - domainNamespaces.add(domainNamespace1); - domainNamespaces.add(domainNamespace2); - opParams = opParams.domainNamespaces(domainNamespaces).enableClusterRoleBinding(enableClusterRoleBinding); - upgradeAndVerifyOperator(opNamespace, opParams); - - logger.info("verify NamespaceWatchingStarted event is logged in namespace {0}", domainNamespace2); - checkEvent(opNamespace, domainNamespace2, null, NAMESPACE_WATCHING_STARTED, "Normal", timestamp); - - timestamp = now(); - - logger.info("Removing domain namespace {0} in the operator watch list", domainNamespace2); - domainNamespaces.clear(); - domainNamespaces.add(domainNamespace1); - opParams = opParams.domainNamespaces(domainNamespaces); - upgradeAndVerifyOperator(opNamespace, opParams); - - logger.info("verify NamespaceWatchingStopped event is logged in namespace {0}", domainNamespace2); - checkNamespaceWatchingStoppedEvent(opNamespace, domainNamespace2, null, "Normal", timestamp, - enableClusterRoleBinding); - } - - /** - * Test verifies the operator logs a NamespaceWatchingStarted event in the respective domain namespace - * when it starts watching a new domain namespace with domainNamespaceSelectionStrategy set to LabelSelector and - * operator logs a NamespaceWatchingStopped event in the respective domain namespace - * when it stops watching a domain namespace. - * If set to LabelSelector, then the operator will manage the set of namespaces discovered by a list of namespaces - * using the value specified by domainNamespaceLabelSelector as a label selector. - * The test upgrades the operator instance through helm to add or remove another domain namespace - * in the operator watch list. - *

- *

{@literal
-   * helm upgrade weblogic-operator kubernetes/charts/weblogic-operator
-   * --namespace ns-ipqy
-   * --reuse-values
-   * --set "domainNamespaceSelectionStrategy=LabelSelector"
-   * --set "domainNamespaceLabelSelector=weblogic-operator\=enabled"
-   * }
-   * 
- *

- */ - @Order(15) - @ParameterizedTest - @ValueSource(booleans = { true, false }) - void testK8SEventsStartStopWatchingNSWithLabelSelector(boolean enableClusterRoleBinding) { - logger.info("testing testK8SEventsStartStopWatchingNSWithLabelSelector with enableClusterRoleBinding={0}", - enableClusterRoleBinding); - OffsetDateTime timestamp = now(); - - logger.info("Labeling namespace {0} to enable it in the operator watch list", domainNamespace3); - // label domainNamespace3 - new Command() - .withParams(new CommandParams() - .command("kubectl label ns " + domainNamespace3 + " weblogic-operator=enabled --overwrite")) - .execute(); - - // Helm upgrade parameters - opParams = opParams - .domainNamespaceSelectionStrategy("LabelSelector") - .domainNamespaceLabelSelector("weblogic-operator=enabled") - .enableClusterRoleBinding(enableClusterRoleBinding); - upgradeAndVerifyOperator(opNamespace, opParams); - - logger.info("verify NamespaceWatchingStarted event is logged in namespace {0}", domainNamespace3); - checkEvent(opNamespace, domainNamespace3, null, NAMESPACE_WATCHING_STARTED, "Normal", timestamp); - - // verify there is no event logged in domainNamespace4 - logger.info("verify NamespaceWatchingStarted event is not logged in {0}", domainNamespace4); - assertFalse(domainEventExists(opNamespace, domainNamespace4, null, NAMESPACE_WATCHING_STARTED, - "Normal", timestamp), "domain event " + NAMESPACE_WATCHING_STARTED + " is logged in " - + domainNamespace4 + ", expected no such event will be logged"); - - timestamp = now(); - logger.info("Labelling namespace {0} to \"weblogic-operator=disabled\" to disable it in the operator " - + "watch list", domainNamespace3); - - // label domainNamespace3 to weblogic-operator=disabled - new Command() - .withParams(new CommandParams() - .command("kubectl label ns " + domainNamespace3 + " weblogic-operator=disabled --overwrite")) - .execute(); - - logger.info("verify NamespaceWatchingStopped event is logged in namespace {0}", domainNamespace3); - checkNamespaceWatchingStoppedEvent(opNamespace, domainNamespace3, null, "Normal", timestamp, - enableClusterRoleBinding); - - if (enableClusterRoleBinding) { - String newNSWithoutLabels = "ns-newnamespace1"; - String newNSWithLabels = "ns-newnamespace2"; - - assertDoesNotThrow(() -> createNamespaces(newNSWithoutLabels, newNSWithLabels), - "Failed to create new namespaces"); - - new Command() - .withParams(new CommandParams() - .command("kubectl label ns " + newNSWithLabels + " weblogic-operator=enabled --overwrite")) - .execute(); - - logger.info("verify NamespaceWatchingStarted event is logged in namespace {0}", newNSWithLabels); - checkEvent(opNamespace, newNSWithLabels, null, NAMESPACE_WATCHING_STARTED, "Normal", timestamp); - - // verify there is no event logged in domainNamespace4 - logger.info("verify NamespaceWatchingStarted event is not logged in {0}", domainNamespace4); - assertFalse(domainEventExists(opNamespace, newNSWithoutLabels, null, NAMESPACE_WATCHING_STARTED, - "Normal", timestamp), "domain event " + NAMESPACE_WATCHING_STARTED + " is logged in " - + newNSWithoutLabels + ", expected no such event will be logged"); - } - } - - private void createNamespaces(String newNSWithoutLabels, String newNSWithLabels) throws ApiException { - createNamespace(newNSWithoutLabels); - createNamespace(newNSWithLabels); - } - - /** - * Test verifies the operator logs a NamespaceWatchingStarted event in the respective domain namespace - * when it starts watching a new domain namespace with domainNamespaceSelectionStrategy set to RegExp and - * operator logs a NamespaceWatchingStopped event in the respective domain namespace - * when it stops watching a domain namespace. - * If set to RegExp, then the operator will manage the set of namespaces discovered by a list of namespaces - * using the value specified by domainNamespaceRegExp as a regular expression matched against the namespace names. - * The test upgrades the operator instance through helm to add or remove another domain namespace - * in the operator watch list. - *

- *

{@literal
-   * helm upgrade weblogic-operator kubernetes/charts/weblogic-operator
-   * --namespace ns-ipqy
-   * --reuse-values
-   * --set "domainNamespaceSelectionStrategy=RegExp"
-   * --set "domainNamespaceRegExp=abcd"
-   * }
-   * 
- *

- */ - @Order(16) - @ParameterizedTest - @ValueSource(booleans = { true, false }) - void testK8SEventsStartStopWatchingNSWithRegExp(boolean enableClusterRoleBinding) { - OffsetDateTime timestamp = now(); - logger.info("Adding a new domain namespace {0} in the operator watch list", domainNamespace5); - // Helm upgrade parameters - opParams = opParams - .domainNamespaceSelectionStrategy("RegExp") - .domainNamespaceRegExp(domainNamespace5.substring(3)) - .enableClusterRoleBinding(enableClusterRoleBinding); - - upgradeAndVerifyOperator(opNamespace, opParams); - - logger.info("verify NamespaceWatchingStarted event is logged in {0}", domainNamespace5); - checkEvent(opNamespace, domainNamespace5, null, NAMESPACE_WATCHING_STARTED, "Normal", timestamp); - - // verify there is no event logged in domainNamespace4 - logger.info("verify NamespaceWatchingStarted event is not logged in {0}", domainNamespace4); - assertFalse(domainEventExists(opNamespace, domainNamespace4, null, NAMESPACE_WATCHING_STARTED, - "Normal", timestamp), "domain event " + NAMESPACE_WATCHING_STARTED + " is logged in " - + domainNamespace4 + ", expected no such event will be logged"); - - timestamp = now(); - logger.info("Setting the domainNamesoaceRegExp to a new value {0}", domainNamespace4.substring(3)); - - // Helm upgrade parameters - opParams = opParams - .domainNamespaceSelectionStrategy("RegExp") - .domainNamespaceRegExp(domainNamespace4.substring(3)); - - upgradeAndVerifyOperator(opNamespace, opParams); - - logger.info("verify NamespaceWatchingStopped event is logged in namespace {0}", domainNamespace5); - checkNamespaceWatchingStoppedEvent(opNamespace, domainNamespace5, null, "Normal", timestamp, - enableClusterRoleBinding); - - logger.info("verify NamespaceWatchingStarted event is logged in namespace {0}", domainNamespace4); - checkEvent(opNamespace, domainNamespace4, null, NAMESPACE_WATCHING_STARTED, "Normal", timestamp); - } - - /** - * Operator helm parameter domainNamespaceSelectionStrategy is set to Dedicated. - * If set to Dedicated, then operator will manage WebLogic Domains only in the same namespace which the operator - * itself is deployed, which is the namespace of the Helm release. - * Operator logs a NamespaceWatchingStopped in the operator domain namespace and - * NamespaceWatchingStopped event in the other domain namespaces when it stops watching a domain namespace. - * - * Test verifies NamespaceWatchingStopped event is logged when operator stops watching a domain namespace. - */ - @Order(17) - @Test - void testK8SEventsStartStopWatchingNSWithDedicated() { - OffsetDateTime timestamp = now(); - - // Helm upgrade parameters - opParams = opParams.domainNamespaceSelectionStrategy("Dedicated") - .enableClusterRoleBinding(false); - - upgradeAndVerifyOperator(opNamespace, opParams); - - logger.info("verify NamespaceWatchingStarted event is logged in {0}", opNamespace); - checkEvent(opNamespace, opNamespace, null, NAMESPACE_WATCHING_STARTED, "Normal", timestamp); - - logger.info("verify NamespaceWatchingStopped event is logged in {0}", domainNamespace4); - checkNamespaceWatchingStoppedEvent(opNamespace, domainNamespace4, null, "Normal", timestamp, false); + checkEvent(opNamespace, domainNamespace2, domainUid, DOMAIN_DELETED, "Normal", timestamp); } /** @@ -956,8 +669,9 @@ void testK8SEventsStartStopWatchingNSWithDedicated() { @AfterAll public static void tearDown() { if (!SKIP_CLEANUP) { - deletePersistentVolumeClaim(domainNamespace1, "sample-pvc"); + deletePersistentVolumeClaim(domainNamespace3, "sample-pvc"); deletePersistentVolume("sample-pv"); + shutdownDomain(domainUid, domainNamespace3); } } @@ -1004,20 +718,43 @@ private static void checkEventWithCount( } // Create and start a WebLogic domain in PV - private void createDomain() { + private static void createDomain(String domainNamespace, String domainUid, String pvName, String pvcName) { + + assertDoesNotThrow(() -> createDomain(domainNamespace, domainUid, pvName, pvcName, + "IF_NEEDED"), + "Failed to create domain custom resource"); + + // verify the admin server service created + checkPodReadyAndServiceExists(adminServerPodName, domainUid, domainNamespace); + + // verify managed server services created + for (int i = 1; i <= replicaCount; i++) { + logger.info("Checking managed server service/pod {0} is created in namespace {1}", + managedServerPodNamePrefix + i, domainNamespace); + checkPodReadyAndServiceExists(managedServerPodNamePrefix + i, domainUid, domainNamespace); + } + } + + // Create and start a WebLogic domain in PV + private static Domain createDomain(String domainNamespace, String domainUid, + String pvName, String pvcName, String serverStartupPolicy) { + + // create pull secrets for WebLogic image when running in non Kind Kubernetes cluster + // this secret is used only for non-kind cluster + createSecretForBaseImages(domainNamespace); // create WebLogic domain credential secret - createSecretWithUsernamePassword(wlSecretName, domainNamespace1, + createSecretWithUsernamePassword(wlSecretName, domainNamespace, ADMIN_USERNAME_DEFAULT, ADMIN_PASSWORD_DEFAULT); // create persistent volume and persistent volume claim for domain // these resources should be labeled with domainUid for cleanup after testing - createPV(pvName, domainUid, this.getClass().getSimpleName()); - createPVC(pvName, pvcName, domainUid, domainNamespace1); - + createPV(pvName, domainUid, className); + createPVC(pvName, pvcName, domainUid, domainNamespace); + int t3ChannelPort = getNextFreePort(); // create a temporary WebLogic domain property file File domainPropertiesFile = assertDoesNotThrow(() - -> File.createTempFile("domain", "properties"), + -> File.createTempFile("domain", "properties"), "Failed to create domain properties file"); Properties p = new Properties(); p.setProperty("domain_path", "/shared/domains"); @@ -1029,13 +766,13 @@ private void createDomain() { p.setProperty("admin_username", ADMIN_USERNAME_DEFAULT); p.setProperty("admin_password", ADMIN_PASSWORD_DEFAULT); p.setProperty("admin_t3_public_address", K8S_NODEPORT_HOST); - p.setProperty("admin_t3_channel_port", Integer.toString(32000)); + p.setProperty("admin_t3_channel_port", Integer.toString(t3ChannelPort)); p.setProperty("number_of_ms", "2"); p.setProperty("managed_server_name_base", managedServerNameBase); p.setProperty("domain_logs", "/shared/logs"); p.setProperty("production_mode_enabled", "true"); assertDoesNotThrow(() - -> p.store(new FileOutputStream(domainPropertiesFile), "domain properties file"), + -> p.store(new FileOutputStream(domainPropertiesFile), "domain properties file"), "Failed to write domain properties file"); // WLST script for creating domain @@ -1043,7 +780,7 @@ private void createDomain() { // create configmap and domain on persistent volume using the WLST script and property file createDomainOnPVUsingWlst(wlstScript, domainPropertiesFile.toPath(), - pvName, pvcName, domainNamespace1); + pvName, pvcName, domainNamespace); // create a domain custom resource configuration object logger.info("Creating domain custom resource"); @@ -1052,7 +789,7 @@ private void createDomain() { .kind("Domain") .metadata(new V1ObjectMeta() .name(domainUid) - .namespace(domainNamespace1)) + .namespace(domainNamespace)) .spec(new DomainSpec() .domainUid(domainUid) .domainHome("/shared/domains/" + domainUid) // point to domain home in pv @@ -1064,12 +801,12 @@ private void createDomain() { .name(BASE_IMAGES_REPO_SECRET))) // this secret is used only in non-kind cluster .webLogicCredentialsSecret(new V1SecretReference() .name(wlSecretName) - .namespace(domainNamespace1)) + .namespace(domainNamespace)) .includeServerOutInPodLog(true) .logHomeEnabled(Boolean.TRUE) .logHome("/shared/logs/" + domainUid) .dataHome("") - .serverStartPolicy("IF_NEEDED") + .serverStartPolicy(serverStartupPolicy) .serverPod(new ServerPod() //serverpod .addEnvItem(new V1EnvVar() .name("USER_MEM_ARGS") @@ -1092,33 +829,23 @@ private void createDomain() { .replicas(replicaCount) .serverStartState("RUNNING"))); setPodAntiAffinity(domain); - // verify the domain custom resource is created - createDomainAndVerify(domain, domainNamespace1); - - // verify the admin server service created - checkPodReadyAndServiceExists(adminServerPodName, domainUid, domainNamespace1); - - // verify managed server services created - for (int i = 1; i <= replicaCount; i++) { - logger.info("Checking managed server service/pod {0} is created in namespace {1}", - managedServerPodNamePrefix + i, domainNamespace1); - checkPodReadyAndServiceExists(managedServerPodNamePrefix + i, domainUid, domainNamespace1); - } + createDomainAndVerify(domain, domainNamespace); + return domain; } /** * Create a WebLogic domain on a persistent volume by doing the following. Create a configmap containing WLST script * and property file. Create a Kubernetes job to create domain on persistent volume. * - * @param wlstScriptFile python script to create domain + * @param wlstScriptFile python script to create domain * @param domainPropertiesFile properties file containing domain configuration - * @param pvName name of the persistent volume to create domain in - * @param pvcName name of the persistent volume claim - * @param namespace name of the domain namespace in which the job is created + * @param pvName name of the persistent volume to create domain in + * @param pvcName name of the persistent volume claim + * @param namespace name of the domain namespace in which the job is created */ - private void createDomainOnPVUsingWlst(Path wlstScriptFile, Path domainPropertiesFile, - String pvName, String pvcName, String namespace) { + private static void createDomainOnPVUsingWlst(Path wlstScriptFile, Path domainPropertiesFile, + String pvName, String pvcName, String namespace) { logger.info("Preparing to run create domain job using WLST"); List domainScriptFiles = new ArrayList<>(); @@ -1129,7 +856,7 @@ private void createDomainOnPVUsingWlst(Path wlstScriptFile, Path domainPropertie String domainScriptConfigMapName = "create-domain-scripts-cm"; assertDoesNotThrow( () -> createConfigMapForDomainCreation( - domainScriptConfigMapName, domainScriptFiles, namespace, this.getClass().getSimpleName()), + domainScriptConfigMapName, domainScriptFiles, namespace, className), "Create configmap for domain creation failed"); // create a V1Container with specific scripts and properties for creating domain @@ -1146,13 +873,13 @@ private void createDomainOnPVUsingWlst(Path wlstScriptFile, Path domainPropertie namespace, jobCreationContainer); } - private void createNewCluster() { + private void createNewCluster(String domainNamespace) { final String managedServerNameBase = "cl2-ms-"; String managedServerPodNamePrefix = domainUid + "-" + managedServerNameBase; logger.info("Getting port for default channel"); int adminServerPort - = getServicePort(domainNamespace1, getExternalServicePodName(adminServerPodName), "default"); + = getServicePort(domainNamespace, getExternalServicePodName(adminServerPodName), "default"); // create a temporary WebLogic WLST property file File wlstPropertiesFile = assertDoesNotThrow(() -> File.createTempFile("wlst", "properties"), @@ -1171,9 +898,9 @@ private void createNewCluster() { // changet the admin server port to a different value to force pod restart Path configScript = Paths.get(RESOURCE_DIR, "python-scripts", "introspect_version_script.py"); - executeWLSTScript(configScript, wlstPropertiesFile.toPath(), domainNamespace1); + executeWLSTScript(configScript, wlstPropertiesFile.toPath(), domainNamespace); - String introspectVersion = assertDoesNotThrow(() -> getNextIntrospectVersion(domainUid, domainNamespace1)); + String introspectVersion = assertDoesNotThrow(() -> getNextIntrospectVersion(domainUid, domainNamespace)); logger.info("patch the domain resource with new cluster and introspectVersion"); String patchStr @@ -1185,47 +912,85 @@ private void createNewCluster() { + "]"; logger.info("Updating domain configuration using patch string: {0}\n", patchStr); V1Patch patch = new V1Patch(patchStr); - assertTrue(patchDomainCustomResource(domainUid, domainNamespace1, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), + assertTrue(patchDomainCustomResource(domainUid, domainNamespace, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), "Failed to patch domain"); //verify the introspector pod is created and runs String introspectPodNameBase = getIntrospectJobName(domainUid); - checkPodExists(introspectPodNameBase, domainUid, domainNamespace1); - checkPodDoesNotExist(introspectPodNameBase, domainUid, domainNamespace1); + checkPodExists(introspectPodNameBase, domainUid, domainNamespace); + checkPodDoesNotExist(introspectPodNameBase, domainUid, domainNamespace); // verify new cluster managed server services created for (int i = 1; i <= replicaCount; i++) { logger.info("Checking managed server service {0} is created in namespace {1}", - managedServerPodNamePrefix + i, domainNamespace1); - checkServiceExists(managedServerPodNamePrefix + i, domainNamespace1); + managedServerPodNamePrefix + i, domainNamespace); + checkServiceExists(managedServerPodNamePrefix + i, domainNamespace); } // verify new cluster managed server pods are ready for (int i = 1; i <= replicaCount; i++) { logger.info("Waiting for managed server pod {0} to be ready in namespace {1}", - managedServerPodNamePrefix + i, domainNamespace1); - checkPodReady(managedServerPodNamePrefix + i, domainUid, domainNamespace1); + managedServerPodNamePrefix + i, domainNamespace); + checkPodReady(managedServerPodNamePrefix + i, domainUid, domainNamespace); } } - private void scaleDomainAndVerifyCompletedEvent(int replicaCount, String testType, boolean verify) { + private void scaleDomainAndVerifyCompletedEvent(int replicaCount, ScaleAction testType, + boolean verify, String namespace) { OffsetDateTime timestamp = now(); logger.info("Updating domain resource to set the replicas for cluster " + cluster1Name + " to " + replicaCount); - int countBefore = getDomainEventCount(domainNamespace1, domainUid, DOMAIN_PROCESSING_COMPLETED, "Normal"); + int countBefore = getDomainEventCount(namespace, domainUid, DOMAIN_PROCESSING_COMPLETED, "Normal"); V1Patch patch = new V1Patch("[" + "{\"op\": \"replace\", \"path\": \"/spec/clusters/0/replicas\", \"value\": " + replicaCount + "}" + "]"); - assertTrue(patchDomainCustomResource(domainUid, domainNamespace1, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), + assertTrue(patchDomainCustomResource(domainUid, namespace, patch, V1Patch.PATCH_FORMAT_JSON_PATCH), "Failed to patch domain"); + int serverNumber = replicaCount + 1; + + switch (testType) { + case scaleUp: + checkPodReady(managedServerPodNamePrefix + replicaCount, domainUid, namespace); + break; + case scaleDown: + checkPodDeleted(managedServerPodNamePrefix + serverNumber, domainUid, namespace); + break; + case reset: + checkPodReady(managedServerPodNamePrefix + replicaCount, domainUid, namespace); + checkPodDeleted(managedServerPodNamePrefix + serverNumber, domainUid, namespace); + break; + default: + } + if (verify) { - logger.info("Verify the DomainProcessingCompleted event is generated after " + testType); + logger.info("Verify the Completed event is generated after " + testType); checkEventWithCount( - opNamespace, domainNamespace1, domainUid, DOMAIN_PROCESSING_COMPLETED, "Normal", timestamp, countBefore); + opNamespace, namespace, domainUid, DOMAIN_PROCESSING_COMPLETED, "Normal", timestamp, countBefore); } } - private void scaleDomain(int replicaCount) { - scaleDomainAndVerifyCompletedEvent(replicaCount, null, false); + private static void checkFailedEvent( + String opNamespace, String domainNamespace, String domainUid, + String failureReason, String type, OffsetDateTime timestamp) { + testUntil(withLongRetryPolicy, + checkDomainFailedEventWithReason(opNamespace, domainNamespace, domainUid, failureReason, type, timestamp), + logger, + "domain event {0} to be logged in namespace {1}", + failureReason, + domainNamespace); } + private void verifyDomainRollAndPodCycleEvents(OffsetDateTime timestamp, String domainNamespace) { + logger.info("verify domain roll starting/pod cycle starting events are logged"); + checkEvent(opNamespace, domainNamespace, domainUid, DOMAIN_ROLL_STARTING, "Normal", timestamp); + checkEvent(opNamespace, domainNamespace, domainUid, POD_CYCLE_STARTING, "Normal", timestamp); + checkEvent(opNamespace, domainNamespace, domainUid, DOMAIN_ROLL_COMPLETED, "Normal", timestamp); + // verify that Rolling condition is removed + testUntil( + () -> verifyDomainStatusConditionTypeDoesNotExist( + domainUid, domainNamespace, "Rolling"), + logger, + "Verifying domain {0} in namespace {1} no longer has a Rolling status condition", + domainUid, + domainNamespace); + } } diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesNameSpaceWatchingEvents.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesNameSpaceWatchingEvents.java new file mode 100644 index 00000000000..17a6bc00cca --- /dev/null +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesNameSpaceWatchingEvents.java @@ -0,0 +1,357 @@ +// Copyright (c) 2022, Oracle and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + + +package oracle.weblogic.kubernetes; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; + +import io.kubernetes.client.openapi.ApiException; +import oracle.weblogic.kubernetes.actions.impl.OperatorParams; +import oracle.weblogic.kubernetes.actions.impl.primitive.Command; +import oracle.weblogic.kubernetes.actions.impl.primitive.CommandParams; +import oracle.weblogic.kubernetes.annotations.IntegrationTest; +import oracle.weblogic.kubernetes.annotations.Namespaces; +import oracle.weblogic.kubernetes.logging.LoggingFacade; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static oracle.weblogic.kubernetes.actions.TestActions.createNamespace; +import static oracle.weblogic.kubernetes.actions.TestActions.now; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntil; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.withLongRetryPolicy; +import static oracle.weblogic.kubernetes.utils.ImageUtils.createSecretForBaseImages; +import static oracle.weblogic.kubernetes.utils.K8sEvents.NAMESPACE_WATCHING_STARTED; +import static oracle.weblogic.kubernetes.utils.K8sEvents.checkDomainEvent; +import static oracle.weblogic.kubernetes.utils.K8sEvents.checkDomainEventWatchingStopped; +import static oracle.weblogic.kubernetes.utils.K8sEvents.domainEventExists; +import static oracle.weblogic.kubernetes.utils.OKDUtils.createRouteForOKD; +import static oracle.weblogic.kubernetes.utils.OKDUtils.setTlsTerminationForRoute; +import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator; +import static oracle.weblogic.kubernetes.utils.OperatorUtils.upgradeAndVerifyOperator; +import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Tests related to Domain events logged by operator. + * The tests checks for the following events in the domain name space. + * NamespaceWatchingStarted, and NamespaceWatchingStopped. + */ +@DisplayName("Verify the Kubernetes events for watching namespace") +@IntegrationTest +class ItKubernetesNameSpaceWatchingEvents { + + private static String opNamespace = null; + private static String domainNamespace1 = null; + private static String domainNamespace2 = null; + private static String domainNamespace3 = null; + private static String domainNamespace4 = null; + private static String domainNamespace5 = null; + private static String opServiceAccount = null; + private static OperatorParams opParams = null; + private static LoggingFacade logger = null; + private static OperatorParams opParamsOriginal = null; + + /** + * Assigns unique namespaces for operator and domains. + * Pull WebLogic image if running tests in Kind cluster. + * Installs operator. + * + * @param namespaces injected by JUnit + */ + @BeforeAll + public static void initAll(@Namespaces(6) List namespaces) { + logger = getLogger(); + logger.info("Assign a unique namespace for operator"); + assertNotNull(namespaces.get(0), "Namespace is null"); + opNamespace = namespaces.get(0); + logger.info("Assign a unique namespace for WebLogic domain"); + assertNotNull(namespaces.get(1), "Namespace is null"); + domainNamespace1 = namespaces.get(1); + assertNotNull(namespaces.get(2), "Namespace is null"); + domainNamespace2 = namespaces.get(2); + assertNotNull(namespaces.get(3), "Namespace is null"); + domainNamespace3 = namespaces.get(3); + assertNotNull(namespaces.get(4), "Namespace is null"); + domainNamespace4 = namespaces.get(4); + assertNotNull(namespaces.get(5), "Namespace is null"); + domainNamespace5 = namespaces.get(5); + + // set the service account name for the operator + opServiceAccount = opNamespace + "-sa"; + + // install and verify operator with REST API + opParams = installAndVerifyOperator(opNamespace, opServiceAccount, true, 0, domainNamespace1); + opParamsOriginal = opParams; + // This test uses the operator restAPI to scale the domain. To do this in OKD cluster, + // we need to expose the external service as route and set tls termination to passthrough + logger.info("Create a route for the operator external service - only for OKD"); + String opExternalSvc = createRouteForOKD("external-weblogic-operator-svc", opNamespace); + // Patch the route just created to set tls termination to passthrough + setTlsTerminationForRoute("external-weblogic-operator-svc", opNamespace); + + // create pull secrets for WebLogic image when running in non Kind Kubernetes cluster + // this secret is used only for non-kind cluster + createSecretForBaseImages(domainNamespace1); + } + + /** + * Test verifies the operator logs a NamespaceWatchingStarted event in the respective domain namespace + * when it starts watching a new domain namespace with domainNamespaceSelectionStrategy default to List and + * operator logs a NamespaceWatchingStopped event in the respective domain namespace + * when it stops watching a domain namespace. + * The test upgrades the operator instance through helm to add or remove another domain namespace + * in the operator watch list. + * This is a parameterized test with enableClusterRoleBinding set to either true or false. + *

+ *

{@literal
+   * helm upgrade weblogic-operator kubernetes/charts/weblogic-operator
+   * --namespace ns-ipqy
+   * --reuse-values
+   * --set "domainNamespaces={ns-xghr,ns-idir}"
+   * }
+   * 
+ *

+ */ + @ParameterizedTest + @ValueSource(booleans = { true, false }) + void testK8SEventsStartStopWatchingNS(boolean enableClusterRoleBinding) { + logger.info("testing testK8SEventsStartStopWatchingNS with enableClusterRoleBinding={0}", + enableClusterRoleBinding); + OffsetDateTime timestamp = now(); + + logger.info("Adding a new domain namespace in the operator watch list"); + List domainNamespaces = new ArrayList<>(); + domainNamespaces.add(domainNamespace1); + domainNamespaces.add(domainNamespace2); + OperatorParams opTestParams = opParamsOriginal; + opTestParams = opTestParams.domainNamespaces(domainNamespaces).enableClusterRoleBinding(enableClusterRoleBinding) + .domainNamespaceSelectionStrategy("List"); + upgradeAndVerifyOperator(opNamespace, opTestParams); + + logger.info("verify NamespaceWatchingStarted event is logged in namespace {0}", domainNamespace2); + checkEvent(opNamespace, domainNamespace2, null, NAMESPACE_WATCHING_STARTED, "Normal", timestamp); + + timestamp = now(); + + logger.info("Removing domain namespace {0} in the operator watch list", domainNamespace2); + domainNamespaces.clear(); + domainNamespaces.add(domainNamespace1); + opTestParams = opTestParams.domainNamespaces(domainNamespaces).domainNamespaceSelectionStrategy("List"); + upgradeAndVerifyOperator(opNamespace, opTestParams); + + logger.info("verify NamespaceWatchingStopped event is logged in namespace {0}", domainNamespace2); + checkNamespaceWatchingStoppedEvent(opNamespace, domainNamespace2, null, "Normal", timestamp, + enableClusterRoleBinding); + } + + /** + * Test verifies the operator logs a NamespaceWatchingStarted event in the respective domain namespace + * when it starts watching a new domain namespace with domainNamespaceSelectionStrategy set to LabelSelector and + * operator logs a NamespaceWatchingStopped event in the respective domain namespace + * when it stops watching a domain namespace. + * If set to LabelSelector, then the operator will manage the set of namespaces discovered by a list of namespaces + * using the value specified by domainNamespaceLabelSelector as a label selector. + * The test upgrades the operator instance through helm to add or remove another domain namespace + * in the operator watch list. + *

+ *

{@literal
+   * helm upgrade weblogic-operator kubernetes/charts/weblogic-operator
+   * --namespace ns-ipqy
+   * --reuse-values
+   * --set "domainNamespaceSelectionStrategy=LabelSelector"
+   * --set "domainNamespaceLabelSelector=weblogic-operator\=enabled"
+   * }
+   * 
+ *

+ */ + @ParameterizedTest + @ValueSource(booleans = { true, false }) + void testK8SEventsStartStopWatchingNSWithLabelSelector(boolean enableClusterRoleBinding) { + logger.info("testing testK8SEventsStartStopWatchingNSWithLabelSelector with enableClusterRoleBinding={0}", + enableClusterRoleBinding); + OffsetDateTime timestamp = now(); + + logger.info("Labeling namespace {0} to enable it in the operator watch list", domainNamespace3); + // label domainNamespace3 + Command + .withParams(new CommandParams() + .command("kubectl label ns " + domainNamespace3 + " weblogic-operator=enabled --overwrite")) + .execute(); + OperatorParams opTestParams = opParamsOriginal; + // Helm upgrade parameters + opTestParams = opTestParams + .domainNamespaceSelectionStrategy("LabelSelector") + .domainNamespaceLabelSelector("weblogic-operator=enabled") + .enableClusterRoleBinding(enableClusterRoleBinding); + upgradeAndVerifyOperator(opNamespace, opTestParams); + + logger.info("verify NamespaceWatchingStarted event is logged in namespace {0}", domainNamespace3); + checkEvent(opNamespace, domainNamespace3, null, NAMESPACE_WATCHING_STARTED, "Normal", timestamp); + + // verify there is no event logged in domainNamespace4 + logger.info("verify NamespaceWatchingStarted event is not logged in {0}", domainNamespace4); + assertFalse(domainEventExists(opNamespace, domainNamespace4, null, NAMESPACE_WATCHING_STARTED, + "Normal", timestamp), "domain event " + NAMESPACE_WATCHING_STARTED + " is logged in " + + domainNamespace4 + ", expected no such event will be logged"); + + timestamp = now(); + logger.info("Labelling namespace {0} to \"weblogic-operator=disabled\" to disable it in the operator " + + "watch list", domainNamespace3); + + // label domainNamespace3 to weblogic-operator=disabled + Command + .withParams(new CommandParams() + .command("kubectl label ns " + domainNamespace3 + " weblogic-operator=disabled --overwrite")) + .execute(); + + logger.info("verify NamespaceWatchingStopped event is logged in namespace {0}", domainNamespace3); + checkNamespaceWatchingStoppedEvent(opNamespace, domainNamespace3, null, "Normal", timestamp, + enableClusterRoleBinding); + + if (enableClusterRoleBinding) { + String newNSWithoutLabels = "ns-newnamespace1"; + String newNSWithLabels = "ns-newnamespace2"; + + assertDoesNotThrow(() -> createNamespaces(newNSWithoutLabels, newNSWithLabels), + "Failed to create new namespaces"); + + Command + .withParams(new CommandParams() + .command("kubectl label ns " + newNSWithLabels + " weblogic-operator=enabled --overwrite")) + .execute(); + + logger.info("verify NamespaceWatchingStarted event is logged in namespace {0}", newNSWithLabels); + checkEvent(opNamespace, newNSWithLabels, null, NAMESPACE_WATCHING_STARTED, "Normal", timestamp); + + // verify there is no event logged in domainNamespace4 + logger.info("verify NamespaceWatchingStarted event is not logged in {0}", domainNamespace4); + assertFalse(domainEventExists(opNamespace, newNSWithoutLabels, null, NAMESPACE_WATCHING_STARTED, + "Normal", timestamp), "domain event " + NAMESPACE_WATCHING_STARTED + " is logged in " + + newNSWithoutLabels + ", expected no such event will be logged"); + } + } + + private void createNamespaces(String newNSWithoutLabels, String newNSWithLabels) throws ApiException { + createNamespace(newNSWithoutLabels); + createNamespace(newNSWithLabels); + } + + /** + * Test verifies the operator logs a NamespaceWatchingStarted event in the respective domain namespace + * when it starts watching a new domain namespace with domainNamespaceSelectionStrategy set to RegExp and + * operator logs a NamespaceWatchingStopped event in the respective domain namespace + * when it stops watching a domain namespace. + * If set to RegExp, then the operator will manage the set of namespaces discovered by a list of namespaces + * using the value specified by domainNamespaceRegExp as a regular expression matched against the namespace names. + * The test upgrades the operator instance through helm to add or remove another domain namespace + * in the operator watch list. + *

+ *

{@literal
+   * helm upgrade weblogic-operator kubernetes/charts/weblogic-operator
+   * --namespace ns-ipqy
+   * --reuse-values
+   * --set "domainNamespaceSelectionStrategy=RegExp"
+   * --set "domainNamespaceRegExp=abcd"
+   * }
+   * 
+ *

+ */ + @ParameterizedTest + @ValueSource(booleans = { true, false }) + void testK8SEventsStartStopWatchingNSWithRegExp(boolean enableClusterRoleBinding) { + OffsetDateTime timestamp = now(); + logger.info("Adding a new domain namespace {0} in the operator watch list", domainNamespace5); + // Helm upgrade parameters + OperatorParams opTestParams = opParamsOriginal; + opTestParams = opTestParams + .domainNamespaceSelectionStrategy("RegExp") + .domainNamespaceRegExp(domainNamespace5.substring(3)) + .enableClusterRoleBinding(enableClusterRoleBinding); + + upgradeAndVerifyOperator(opNamespace, opTestParams); + + logger.info("verify NamespaceWatchingStarted event is logged in {0}", domainNamespace5); + checkEvent(opNamespace, domainNamespace5, null, NAMESPACE_WATCHING_STARTED, "Normal", timestamp); + + // verify there is no event logged in domainNamespace4 + logger.info("verify NamespaceWatchingStarted event is not logged in {0}", domainNamespace4); + assertFalse(domainEventExists(opNamespace, domainNamespace4, null, NAMESPACE_WATCHING_STARTED, + "Normal", timestamp), "domain event " + NAMESPACE_WATCHING_STARTED + " is logged in " + + domainNamespace4 + ", expected no such event will be logged"); + + timestamp = now(); + logger.info("Setting the domainNamesoaceRegExp to a new value {0}", domainNamespace4.substring(3)); + + // Helm upgrade parameters + opTestParams = opTestParams + .domainNamespaceSelectionStrategy("RegExp") + .domainNamespaceRegExp(domainNamespace4.substring(3)); + + upgradeAndVerifyOperator(opNamespace, opTestParams); + + logger.info("verify NamespaceWatchingStopped event is logged in namespace {0}", domainNamespace5); + checkNamespaceWatchingStoppedEvent(opNamespace, domainNamespace5, null, "Normal", timestamp, + enableClusterRoleBinding); + + logger.info("verify NamespaceWatchingStarted event is logged in namespace {0}", domainNamespace4); + checkEvent(opNamespace, domainNamespace4, null, NAMESPACE_WATCHING_STARTED, "Normal", timestamp); + } + + /** + * Operator helm parameter domainNamespaceSelectionStrategy is set to Dedicated. + * If set to Dedicated, then operator will manage WebLogic Domains only in the same namespace which the operator + * itself is deployed, which is the namespace of the Helm release. + * Operator logs a NamespaceWatchingStopped in the operator domain namespace and + * NamespaceWatchingStopped event in the other domain namespaces when it stops watching a domain namespace. + * + * Test verifies NamespaceWatchingStopped event is logged when operator stops watching a domain namespace. + */ + @Test + void testK8SEventsStartStopWatchingNSWithDedicated() { + OffsetDateTime timestamp = now(); + + // Helm upgrade parameters + OperatorParams opTestParams = opParamsOriginal; + opTestParams = opTestParams.domainNamespaceSelectionStrategy("Dedicated") + .enableClusterRoleBinding(false); + + upgradeAndVerifyOperator(opNamespace, opTestParams); + + logger.info("verify NamespaceWatchingStarted event is logged in {0}", opNamespace); + checkEvent(opNamespace, opNamespace, null, NAMESPACE_WATCHING_STARTED, "Normal", timestamp); + + logger.info("verify NamespaceWatchingStopped event is logged in {0}", domainNamespace4); + checkNamespaceWatchingStoppedEvent(opNamespace, domainNamespace4, null, "Normal", timestamp, false); + } + + // Utility method to check event + private static void checkEvent( + String opNamespace, String domainNamespace, String domainUid, + String reason, String type, OffsetDateTime timestamp) { + testUntil(withLongRetryPolicy, + checkDomainEvent(opNamespace, domainNamespace, domainUid, reason, type, timestamp), + logger, + "domain event {0} to be logged in namespace {1}", + reason, + domainNamespace); + } + + private static void checkNamespaceWatchingStoppedEvent( + String opNamespace, String domainNamespace, String domainUid, + String type, OffsetDateTime timestamp, boolean enableClusterRoleBinding) { + testUntil( + checkDomainEventWatchingStopped( + opNamespace, domainNamespace, domainUid, type, timestamp, enableClusterRoleBinding), + logger, + "domain event NamespaceWatchingStopped to be logged in namespace {0}", + domainNamespace); + } +} diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/TestActions.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/TestActions.java index 7af91172d32..7d72f3b232c 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/TestActions.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/TestActions.java @@ -203,6 +203,21 @@ public static oracle.weblogic.domain.Domain getDomainCustomResource(String domai return Domain.getDomainCustomResource(domainUid, namespace); } + /** + * Get the Domain Custom Resource. + * + * @param domainUid unique domain identifier + * @param namespace name of namespace + * @param domainVersion version of domain + * @return Domain Custom Resource or null if Domain does not exist + * @throws ApiException if Kubernetes client API call fails + */ + public static oracle.weblogic.domain.Domain getDomainCustomResource(String domainUid, + String namespace, + String domainVersion) throws ApiException { + return Domain.getDomainCustomResource(domainUid, namespace, domainVersion); + } + /** * Shutdown the domain. * diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/Domain.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/Domain.java index 3ae76f5fb87..ea0422fdea7 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/Domain.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/Domain.java @@ -165,6 +165,21 @@ public static oracle.weblogic.domain.Domain getDomainCustomResource(String domai return Kubernetes.getDomainCustomResource(domainUid, namespace); } + /** + * Get a Domain Custom Resource. + * + * @param domainUid unique domain identifier + * @param namespace name of namespace + * @param domainVersion domain version + * @return domain custom resource or null if Domain does not exist + * @throws ApiException if Kubernetes request fails + */ + public static oracle.weblogic.domain.Domain getDomainCustomResource(String domainUid, + String namespace, + String domainVersion) throws ApiException { + return Kubernetes.getDomainCustomResource(domainUid, namespace, domainVersion); + } + /** * Patch the Domain Custom Resource. * diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/primitive/Kubernetes.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/primitive/Kubernetes.java index 847451ef0ec..8459a0711b2 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/primitive/Kubernetes.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/primitive/Kubernetes.java @@ -1265,6 +1265,39 @@ public static Domain getDomainCustomResource(String domainUid, String namespace) return null; } + /** + * Get the Domain Custom Resource. + * + * @param domainUid unique domain identifier + * @param namespace name of namespace + * @param domainVersion version of domain + * @return domain custom resource or null if Domain does not exist + * @throws ApiException if Kubernetes request fails + */ + public static Domain getDomainCustomResource(String domainUid, String namespace, String domainVersion) + throws ApiException { + Object domain; + try { + domain = customObjectsApi.getNamespacedCustomObject( + DOMAIN_GROUP, // custom resource's group name + domainVersion, // //custom resource's version + namespace, // custom resource's namespace + DOMAIN_PLURAL, // custom resource's plural name + domainUid // custom object's name + ); + } catch (ApiException apex) { + getLogger().severe(apex.getResponseBody()); + throw apex; + } + + if (domain != null) { + return handleResponse(domain, Domain.class); + } + + getLogger().warning("Domain Custom Resource '" + domainUid + "' not found in namespace " + namespace); + return null; + } + /** * Patch the Domain Custom Resource using JSON Patch.JSON Patch is a format for describing changes to a JSON document * using a series of operations. JSON Patch is specified in RFC 6902 from the IETF. For example, the following diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/DomainUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/DomainUtils.java index 18b2021f421..aedf2d83748 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/DomainUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/DomainUtils.java @@ -45,6 +45,7 @@ import oracle.weblogic.domain.Cluster; import oracle.weblogic.domain.Configuration; import oracle.weblogic.domain.Domain; +import oracle.weblogic.domain.DomainCondition; import oracle.weblogic.domain.DomainSpec; import oracle.weblogic.domain.Model; import oracle.weblogic.domain.ServerPod; @@ -80,6 +81,7 @@ import static oracle.weblogic.kubernetes.actions.TestActions.createConfigMap; import static oracle.weblogic.kubernetes.actions.TestActions.createDomainCustomResource; import static oracle.weblogic.kubernetes.actions.TestActions.deleteDomainCustomResource; +import static oracle.weblogic.kubernetes.actions.TestActions.getDomainCustomResource; import static oracle.weblogic.kubernetes.actions.TestActions.getJob; import static oracle.weblogic.kubernetes.actions.TestActions.getPodLog; import static oracle.weblogic.kubernetes.actions.TestActions.listPods; @@ -749,4 +751,68 @@ public static void shutdownDomainAndVerify(String domainNamespace, String domain } } + /** + * Check the domain status condition type does not exist. + * @param domainUid uid of the domain + * @param domainNamespace namespace of the domain + * @param conditionType the type name of condition, accepted value: Completed, Available, Failed and + * ConfigChangesPendingRestart + * @return true if the condition type does not exist, false otherwise + */ + public static boolean verifyDomainStatusConditionTypeDoesNotExist(String domainUid, + String domainNamespace, + String conditionType) { + return verifyDomainStatusConditionTypeDoesNotExist(domainUid, domainNamespace, + conditionType, DOMAIN_VERSION); + } + + /** + * Check the domain status condition type does not exist. + * @param domainUid uid of the domain + * @param domainNamespace namespace of the domain + * @param conditionType the type name of condition, accepted value: Completed, Available, Failed and + * ConfigChangesPendingRestart + * @param domainVersion version of domain + * @return true if the condition type does not exist, false otherwise + */ + public static boolean verifyDomainStatusConditionTypeDoesNotExist(String domainUid, + String domainNamespace, + String conditionType, + String domainVersion) { + Domain domain = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, domainNamespace, + domainVersion)); + + if (domain != null && domain.getStatus() != null) { + List domainConditionList = domain.getStatus().getConditions(); + for (DomainCondition domainCondition : domainConditionList) { + if (domainCondition.getType().equalsIgnoreCase(conditionType)) { + return false; + } + } + } else { + if (domain == null) { + getLogger().info("domain is null"); + } else { + getLogger().info("domain status is null"); + } + } + return true; + } + + /** + * Obtains the specified domain, validates that it has a spec and no rolling condition. + * @param domainNamespace the namespace + * @param domainUid the UID + */ + @org.jetbrains.annotations.NotNull + public static Domain getAndValidateInitialDomain(String domainNamespace, String domainUid) { + Domain domain = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, domainNamespace), + String.format("getDomainCustomResource failed " + + "with ApiException when tried to get domain %s in namespace %s", + domainUid, domainNamespace)); + + assertNotNull(domain, "Got null domain resource"); + assertNotNull(domain.getSpec(), domain + "/spec is null"); + return domain; + } } diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/K8sEvents.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/K8sEvents.java index 18de1256a9c..500a0b4dcc5 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/K8sEvents.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/K8sEvents.java @@ -26,6 +26,7 @@ import static oracle.weblogic.kubernetes.actions.TestActions.getPodLog; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.withStandardRetryPolicy; import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -35,6 +36,25 @@ public class K8sEvents { private static final LoggingFacade logger = getLogger(); + public static final String ABORTED_ERROR = "Domain processing is aborted"; + public static final String DOMAIN_CREATED = "DomainCreated"; + public static final String DOMAIN_DELETED = "DomainDeleted"; + public static final String DOMAIN_CHANGED = "DomainChanged"; + public static final String DOMAIN_FAILED = "DomainFailed"; + public static final String DOMAIN_PROCESSING_STARTING = "DomainProcessingStarting"; + public static final String DOMAIN_PROCESSING_COMPLETED = "DomainProcessingCompleted"; + public static final String DOMAIN_PROCESSING_FAILED = "DomainProcessingFailed"; + public static final String DOMAIN_PROCESSING_RETRYING = "DomainProcessingRetrying"; + public static final String DOMAIN_PROCESSING_ABORTED = "DomainProcessingAborted"; + public static final String DOMAIN_ROLL_STARTING = "DomainRollStarting"; + public static final String DOMAIN_ROLL_COMPLETED = "DomainRollCompleted"; + public static final String DOMAIN_VALIDATION_ERROR = "DomainValidationError"; + public static final String NAMESPACE_WATCHING_STARTED = "NamespaceWatchingStarted"; + public static final String NAMESPACE_WATCHING_STOPPED = "NamespaceWatchingStopped"; + public static final String STOP_MANAGING_NAMESPACE = "StopManagingNamespace"; + public static final String POD_TERMINATED = "Killing"; + public static final String POD_STARTED = "Started"; + public static final String POD_CYCLE_STARTING = "PodCycleStarting"; /** * Utility method to check event. @@ -492,23 +512,54 @@ private static void verifyOperatorDetails( } } + /** + * Check if a given DomainFailed event is logged by the operator. + * + * @param opNamespace namespace in which the operator is running + * @param domainNamespace namespace in which the domain exists + * @param domainUid UID of the domain + * @param failureReason DomainFailureReason to check + * @param type type of event, Normal of Warning + * @param timestamp the timestamp after which to see events + */ + public static Callable checkDomainFailedEventWithReason( + String opNamespace, String domainNamespace, String domainUid, String failureReason, + String type, OffsetDateTime timestamp) { + return () -> { + return domainFailedEventExists(opNamespace, domainNamespace, domainUid, failureReason, type, timestamp); + }; + } - public static final String DOMAIN_CREATED = "DomainCreated"; - public static final String DOMAIN_DELETED = "DomainDeleted"; - public static final String DOMAIN_CHANGED = "DomainChanged"; - public static final String DOMAIN_PROCESSING_STARTING = "DomainProcessingStarting"; - public static final String DOMAIN_PROCESSING_COMPLETED = "DomainProcessingCompleted"; - public static final String DOMAIN_PROCESSING_FAILED = "DomainProcessingFailed"; - public static final String DOMAIN_PROCESSING_RETRYING = "DomainProcessingRetrying"; - public static final String DOMAIN_PROCESSING_ABORTED = "DomainProcessingAborted"; - public static final String DOMAIN_ROLL_STARTING = "DomainRollStarting"; - public static final String DOMAIN_ROLL_COMPLETED = "DomainRollCompleted"; - public static final String DOMAIN_VALIDATION_ERROR = "DomainValidationError"; - public static final String NAMESPACE_WATCHING_STARTED = "NamespaceWatchingStarted"; - public static final String NAMESPACE_WATCHING_STOPPED = "NamespaceWatchingStopped"; - public static final String STOP_MANAGING_NAMESPACE = "StopManagingNamespace"; - public static final String POD_TERMINATED = "Killing"; - public static final String POD_STARTED = "Started"; - public static final String POD_CYCLE_STARTING = "PodCycleStarting"; + /** + * Check if a given event is logged by the operator in the given namespace. + * + * @param opNamespace namespace in which the operator is running + * @param domainNamespace namespace in which the event is logged + * @param domainUid UID of the domain + * @param failureReason failure reason to check + * @param type type of event, Normal or Warning + * @param timestamp the timestamp after which to see events + */ + public static boolean domainFailedEventExists( + String opNamespace, String domainNamespace, String domainUid, String failureReason, + String type, OffsetDateTime timestamp) { + try { + List events = Kubernetes.listOpGeneratedNamespacedEvents(domainNamespace); + for (CoreV1Event event : events) { + if (DOMAIN_FAILED.equals(event.getReason()) && (isEqualOrAfter(timestamp, event)) + && event.getMessage().contains(failureReason)) { + logger.info(Yaml.dump(event)); + verifyOperatorDetails(event, opNamespace, domainUid); + //verify type + logger.info("Verifying domain event type {0}", type); + assertEquals(event.getType(), type); + return true; + } + } + } catch (ApiException ex) { + logger.log(Level.SEVERE, null, ex); + } + return false; + } }