Skip to content

fix intermittent test failures in nightly run in ItMiiAuxiliaryImage #2491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 4, 2021
Merged

fix intermittent test failures in nightly run in ItMiiAuxiliaryImage #2491

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
17d6a6d
fix intermittent test failure in nightly parallel run
xiancao Aug 2, 2021
e4a3225
add private methods for introspector pod log
xiancao Aug 3, 2021
a4ec152
Merge branch 'main' of https://github.com/oracle/weblogic-kubernetes-…
xiancao Aug 3, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 65 additions & 15 deletions integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
import java.util.Map;

import io.kubernetes.client.custom.V1Patch;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.models.V1Pod;
import oracle.weblogic.domain.AuxiliaryImage;
import oracle.weblogic.domain.Domain;
import oracle.weblogic.kubernetes.actions.impl.primitive.Command;
Expand Down Expand Up @@ -59,6 +61,8 @@
import static oracle.weblogic.kubernetes.actions.TestActions.dockerTag;
import static oracle.weblogic.kubernetes.actions.TestActions.getDomainCustomResource;
import static oracle.weblogic.kubernetes.actions.TestActions.getOperatorPodName;
import static oracle.weblogic.kubernetes.actions.TestActions.getPod;
import static oracle.weblogic.kubernetes.actions.TestActions.getPodLog;
import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort;
import static oracle.weblogic.kubernetes.actions.TestActions.now;
import static oracle.weblogic.kubernetes.actions.TestActions.patchDomainCustomResource;
Expand All @@ -79,7 +83,7 @@
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createSecretWithUsernamePassword;
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.deleteDomainResource;
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getExternalServicePodName;
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getIntrospectorPodName;
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getIntrospectJobName;
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getPodCreationTime;
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getPodsWithTimeStamps;
import static oracle.weblogic.kubernetes.utils.FileUtils.copyFileFromPod;
Expand Down Expand Up @@ -476,8 +480,7 @@ public void testErrorPathDomainMismatchMountPath() {

// check the introspector pod log contains the expected error message
String expectedErrorMsg = "Auxiliary Image: Dir '/errorpath' doesn't exist or is empty. Exiting.";
String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace));
checkPodLogContainsString(errorpathDomainNamespace, introspectorPodName, expectedErrorMsg);
verifyIntrospectorPodLogContainsExpectedErrorMsg(domainUid, errorpathDomainNamespace, expectedErrorMsg);

// check the domain event contains the expected error message
checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED,
Expand Down Expand Up @@ -559,9 +562,7 @@ public void testErrorPathDomainMissingWDTBinary() {
String expectedErrorMsg = "The domain resource 'spec.domainHomeSourceType' is 'FromModel' and "
+ "a WebLogic Deploy Tool (WDT) install is not located at 'spec.configuration.model.wdtInstallHome' "
+ "which is currently set to '/auxiliary/weblogic-deploy'";
String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace),
"Can't get introspector pod's name");
checkPodLogContainsString(errorpathDomainNamespace, introspectorPodName, expectedErrorMsg);
verifyIntrospectorPodLogContainsExpectedErrorMsg(domainUid, errorpathDomainNamespace, expectedErrorMsg);

// check the domain event contains the expected error message
checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED,
Expand Down Expand Up @@ -647,9 +648,7 @@ public void testErrorPathDomainMissingDomainConfig() {
// check the introspector pod log contains the expected error message
String expectedErrorMsg =
"createDomain did not find the required domainInfo section in the model file /auxiliary/models/model.jms2.yaml";
String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace),
"Get introspector's pod name failed");
checkPodLogContainsString(errorpathDomainNamespace, introspectorPodName, expectedErrorMsg);
verifyIntrospectorPodLogContainsExpectedErrorMsg(domainUid, errorpathDomainNamespace, expectedErrorMsg);

// check the domain event contains the expected error message
checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED,
Expand Down Expand Up @@ -723,9 +722,7 @@ public void testErrorPathDomainWithFailCustomMountCommand() {

// check the introspector pod log contains the expected error message
String expectedErrorMsg = "Auxiliary Image: Command 'exit 1' execution failed in container";
String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, domainNamespace),
"Get introspector's pod name failed");
checkPodLogContainsString(domainNamespace, introspectorPodName, expectedErrorMsg);
verifyIntrospectorPodLogContainsExpectedErrorMsg(domainUid, domainNamespace, expectedErrorMsg);

// check the domain event contains the expected error message
checkDomainEventContainsExpectedMsg(opNamespace, domainNamespace, domainUid, DOMAIN_PROCESSING_FAILED,
Expand Down Expand Up @@ -845,9 +842,7 @@ public void testErrorPathFilePermission() {

// check the introspector pod log contains the expected error message
String expectedErrorMsg = "cp: can't open '/auxiliary/test1.txt': Permission denied";
String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace),
"Can't get introspector's pod name");
checkPodLogContainsString(errorpathDomainNamespace, introspectorPodName, expectedErrorMsg);
verifyIntrospectorPodLogContainsExpectedErrorMsg(domainUid, errorpathDomainNamespace, expectedErrorMsg);

// check the domain event contains the expected error message
checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED,
Expand Down Expand Up @@ -1218,4 +1213,59 @@ private String checkWDTVersion(String domainNamespace, String auxiliaryImagePath

return Files.readAllLines(Paths.get(RESULTS_ROOT, "/WDTversion.txt")).get(0);
}

private boolean introspectorPodLogContainsExpectedErrorMsg(String domainUid,
String namespace,
String errormsg) {
String introspectPodName;
V1Pod introspectorPod;

String introspectJobName = getIntrospectJobName(domainUid);
String labelSelector = String.format("weblogic.domainUID in (%s)", domainUid);

try {
introspectorPod = getPod(namespace, labelSelector, introspectJobName);
} catch (ApiException apiEx) {
logger.severe("got ApiException while getting pod: {0}", apiEx);
return false;
}

if (introspectorPod != null && introspectorPod.getMetadata() != null) {
introspectPodName = introspectorPod.getMetadata().getName();
logger.info("found introspectore pod {0} in namespace {1}", introspectPodName, namespace);
} else {
return false;
}

String introspectorLog;
try {
introspectorLog = getPodLog(introspectPodName, namespace);
logger.info("introspector log: {0}", introspectorLog);
} catch (ApiException apiEx) {
logger.severe("got ApiException while getting pod log: {0}", apiEx);
return false;
}

return introspectorLog.contains(errormsg);
}

private void verifyIntrospectorPodLogContainsExpectedErrorMsg(String domainUid,
String namespace,
String expectedErrorMsg) {

// wait and check whether the introspector log contains the expected error message
logger.info("verifying that the introspector log contains the expected error message");
withStandardRetryPolicy
.conditionEvaluationListener(
condition ->
logger.info(
"Checking for the log of introspector pod contains the expected error msg {0}. "
+ "Elapsed time {1}ms, remaining time {2}ms",
expectedErrorMsg,
condition.getElapsedTimeInMS(),
condition.getRemainingTimeInMS()))
.until(() ->
introspectorPodLogContainsExpectedErrorMsg(domainUid, namespace, expectedErrorMsg));
}

}