From 17d6a6de511b82d99f3b14ef43b4dc4a685e2d1c Mon Sep 17 00:00:00 2001 From: xiancao Date: Mon, 2 Aug 2021 21:01:04 +0000 Subject: [PATCH 1/2] fix intermittent test failure in nightly parallel run --- .../kubernetes/ItMiiAuxiliaryImage.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java index 1fdd06fcf25..f1414956f9d 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java @@ -476,8 +476,8 @@ public void testErrorPathDomainMismatchMountPath() { // check the introspector pod log contains the expected error message String expectedErrorMsg = "Auxiliary Image: Dir '/errorpath' doesn't exist or is empty. Exiting."; - String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace)); - checkPodLogContainsString(errorpathDomainNamespace, introspectorPodName, expectedErrorMsg); + checkPodLogContainsString(errorpathDomainNamespace, + assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace)), expectedErrorMsg); // check the domain event contains the expected error message checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, @@ -559,9 +559,9 @@ public void testErrorPathDomainMissingWDTBinary() { String expectedErrorMsg = "The domain resource 'spec.domainHomeSourceType' is 'FromModel' and " + "a WebLogic Deploy Tool (WDT) install is not located at 'spec.configuration.model.wdtInstallHome' " + "which is currently set to '/auxiliary/weblogic-deploy'"; - String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace), - "Can't get introspector pod's name"); - checkPodLogContainsString(errorpathDomainNamespace, introspectorPodName, expectedErrorMsg); + checkPodLogContainsString(errorpathDomainNamespace, + assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace), + "Can't get introspector pod's name"), expectedErrorMsg); // check the domain event contains the expected error message checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, @@ -647,9 +647,9 @@ public void testErrorPathDomainMissingDomainConfig() { // check the introspector pod log contains the expected error message String expectedErrorMsg = "createDomain did not find the required domainInfo section in the model file /auxiliary/models/model.jms2.yaml"; - String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace), - "Get introspector's pod name failed"); - checkPodLogContainsString(errorpathDomainNamespace, introspectorPodName, expectedErrorMsg); + checkPodLogContainsString(errorpathDomainNamespace, + assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace), + "Get introspector's pod name failed"), expectedErrorMsg); // check the domain event contains the expected error message checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, @@ -723,9 +723,9 @@ public void testErrorPathDomainWithFailCustomMountCommand() { // check the introspector pod log contains the expected error message String expectedErrorMsg = "Auxiliary Image: Command 'exit 1' execution failed in container"; - String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, domainNamespace), - "Get introspector's pod name failed"); - checkPodLogContainsString(domainNamespace, introspectorPodName, expectedErrorMsg); + checkPodLogContainsString(domainNamespace, + assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, domainNamespace), + "Get introspector's pod name failed"), expectedErrorMsg); // check the domain event contains the expected error message checkDomainEventContainsExpectedMsg(opNamespace, domainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, @@ -845,9 +845,9 @@ public void testErrorPathFilePermission() { // check the introspector pod log contains the expected error message String expectedErrorMsg = "cp: can't open '/auxiliary/test1.txt': Permission denied"; - String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace), - "Can't get introspector's pod name"); - checkPodLogContainsString(errorpathDomainNamespace, introspectorPodName, expectedErrorMsg); + checkPodLogContainsString(errorpathDomainNamespace, + assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace), + "Can't get introspector's pod name"), expectedErrorMsg); // check the domain event contains the expected error message checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, From e4a3225e1ad16c204f8da57c6fc52e3bd1e92318 Mon Sep 17 00:00:00 2001 From: xiancao Date: Tue, 3 Aug 2021 15:02:53 +0000 Subject: [PATCH 2/2] add private methods for introspector pod log --- .../kubernetes/ItMiiAuxiliaryImage.java | 80 +++++++++++++++---- 1 file changed, 65 insertions(+), 15 deletions(-) diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java index f1414956f9d..a230d9eaf15 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java @@ -14,6 +14,8 @@ import java.util.Map; import io.kubernetes.client.custom.V1Patch; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.models.V1Pod; import oracle.weblogic.domain.AuxiliaryImage; import oracle.weblogic.domain.Domain; import oracle.weblogic.kubernetes.actions.impl.primitive.Command; @@ -59,6 +61,8 @@ import static oracle.weblogic.kubernetes.actions.TestActions.dockerTag; import static oracle.weblogic.kubernetes.actions.TestActions.getDomainCustomResource; import static oracle.weblogic.kubernetes.actions.TestActions.getOperatorPodName; +import static oracle.weblogic.kubernetes.actions.TestActions.getPod; +import static oracle.weblogic.kubernetes.actions.TestActions.getPodLog; import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort; import static oracle.weblogic.kubernetes.actions.TestActions.now; import static oracle.weblogic.kubernetes.actions.TestActions.patchDomainCustomResource; @@ -81,7 +85,7 @@ import static oracle.weblogic.kubernetes.utils.CommonTestUtils.deleteDomainResource; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.dockerLoginAndPushImageToRegistry; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getExternalServicePodName; -import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getIntrospectorPodName; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getIntrospectJobName; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getPodCreationTime; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getPodsWithTimeStamps; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyOperator; @@ -476,8 +480,7 @@ public void testErrorPathDomainMismatchMountPath() { // check the introspector pod log contains the expected error message String expectedErrorMsg = "Auxiliary Image: Dir '/errorpath' doesn't exist or is empty. Exiting."; - checkPodLogContainsString(errorpathDomainNamespace, - assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace)), expectedErrorMsg); + verifyIntrospectorPodLogContainsExpectedErrorMsg(domainUid, errorpathDomainNamespace, expectedErrorMsg); // check the domain event contains the expected error message checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, @@ -559,9 +562,7 @@ public void testErrorPathDomainMissingWDTBinary() { String expectedErrorMsg = "The domain resource 'spec.domainHomeSourceType' is 'FromModel' and " + "a WebLogic Deploy Tool (WDT) install is not located at 'spec.configuration.model.wdtInstallHome' " + "which is currently set to '/auxiliary/weblogic-deploy'"; - checkPodLogContainsString(errorpathDomainNamespace, - assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace), - "Can't get introspector pod's name"), expectedErrorMsg); + verifyIntrospectorPodLogContainsExpectedErrorMsg(domainUid, errorpathDomainNamespace, expectedErrorMsg); // check the domain event contains the expected error message checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, @@ -647,9 +648,7 @@ public void testErrorPathDomainMissingDomainConfig() { // check the introspector pod log contains the expected error message String expectedErrorMsg = "createDomain did not find the required domainInfo section in the model file /auxiliary/models/model.jms2.yaml"; - checkPodLogContainsString(errorpathDomainNamespace, - assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace), - "Get introspector's pod name failed"), expectedErrorMsg); + verifyIntrospectorPodLogContainsExpectedErrorMsg(domainUid, errorpathDomainNamespace, expectedErrorMsg); // check the domain event contains the expected error message checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, @@ -723,9 +722,7 @@ public void testErrorPathDomainWithFailCustomMountCommand() { // check the introspector pod log contains the expected error message String expectedErrorMsg = "Auxiliary Image: Command 'exit 1' execution failed in container"; - checkPodLogContainsString(domainNamespace, - assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, domainNamespace), - "Get introspector's pod name failed"), expectedErrorMsg); + verifyIntrospectorPodLogContainsExpectedErrorMsg(domainUid, domainNamespace, expectedErrorMsg); // check the domain event contains the expected error message checkDomainEventContainsExpectedMsg(opNamespace, domainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, @@ -845,9 +842,7 @@ public void testErrorPathFilePermission() { // check the introspector pod log contains the expected error message String expectedErrorMsg = "cp: can't open '/auxiliary/test1.txt': Permission denied"; - checkPodLogContainsString(errorpathDomainNamespace, - assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace), - "Can't get introspector's pod name"), expectedErrorMsg); + verifyIntrospectorPodLogContainsExpectedErrorMsg(domainUid, errorpathDomainNamespace, expectedErrorMsg); // check the domain event contains the expected error message checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, @@ -1218,4 +1213,59 @@ private String checkWDTVersion(String domainNamespace, String auxiliaryImagePath return Files.readAllLines(Paths.get(RESULTS_ROOT, "/WDTversion.txt")).get(0); } + + private boolean introspectorPodLogContainsExpectedErrorMsg(String domainUid, + String namespace, + String errormsg) { + String introspectPodName; + V1Pod introspectorPod; + + String introspectJobName = getIntrospectJobName(domainUid); + String labelSelector = String.format("weblogic.domainUID in (%s)", domainUid); + + try { + introspectorPod = getPod(namespace, labelSelector, introspectJobName); + } catch (ApiException apiEx) { + logger.severe("got ApiException while getting pod: {0}", apiEx); + return false; + } + + if (introspectorPod != null && introspectorPod.getMetadata() != null) { + introspectPodName = introspectorPod.getMetadata().getName(); + logger.info("found introspectore pod {0} in namespace {1}", introspectPodName, namespace); + } else { + return false; + } + + String introspectorLog; + try { + introspectorLog = getPodLog(introspectPodName, namespace); + logger.info("introspector log: {0}", introspectorLog); + } catch (ApiException apiEx) { + logger.severe("got ApiException while getting pod log: {0}", apiEx); + return false; + } + + return introspectorLog.contains(errormsg); + } + + private void verifyIntrospectorPodLogContainsExpectedErrorMsg(String domainUid, + String namespace, + String expectedErrorMsg) { + + // wait and check whether the introspector log contains the expected error message + logger.info("verifying that the introspector log contains the expected error message"); + withStandardRetryPolicy + .conditionEvaluationListener( + condition -> + logger.info( + "Checking for the log of introspector pod contains the expected error msg {0}. " + + "Elapsed time {1}ms, remaining time {2}ms", + expectedErrorMsg, + condition.getElapsedTimeInMS(), + condition.getRemainingTimeInMS())) + .until(() -> + introspectorPodLogContainsExpectedErrorMsg(domainUid, namespace, expectedErrorMsg)); + } + }