From 8c937494cbea7738f8aaa9761920182373517878 Mon Sep 17 00:00:00 2001 From: xiancao Date: Mon, 12 Jul 2021 16:59:45 +0000 Subject: [PATCH 1/8] add error path testing for common mount --- .../kubernetes/ItManageNameSpace.java | 20 +- .../weblogic/kubernetes/ItMiiCommonMount.java | 270 +++++++++++++++++- .../kubernetes/assertions/TestAssertions.java | 26 ++ .../kubernetes/utils/CommonTestUtils.java | 71 +++++ .../weblogic/kubernetes/utils/K8sEvents.java | 2 +- 5 files changed, 367 insertions(+), 22 deletions(-) diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItManageNameSpace.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItManageNameSpace.java index a00870531ba..613eb8fe243 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItManageNameSpace.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItManageNameSpace.java @@ -41,7 +41,6 @@ import static oracle.weblogic.kubernetes.TestConstants.ADMIN_SERVER_NAME_BASE; import static oracle.weblogic.kubernetes.TestConstants.ADMIN_USERNAME_DEFAULT; import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_API_VERSION; -import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_VERSION; import static oracle.weblogic.kubernetes.TestConstants.MANAGED_SERVER_NAME_BASE; import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_NAME; import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_TAG; @@ -58,12 +57,12 @@ import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort; import static oracle.weblogic.kubernetes.actions.TestActions.scaleClusterWithRestApi; import static oracle.weblogic.kubernetes.actions.TestActions.uninstallOperator; -import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainDoesNotExist; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodDoesNotExist; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDomainAndVerify; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createOcirRepoSecret; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createSecretWithUsernamePassword; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.deleteDomainResource; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyOperator; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.scaleAndVerifyCluster; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.setPodAntiAffinity; @@ -382,23 +381,6 @@ private void checkUpgradeFailedToAddNSManagedByAnotherOperator() { + " managed by other operator"); } - private void deleteDomainResource(String domainNS, String domainUid) { - //clean up domain resources in namespace and set namespace to label , managed by operator - logger.info("deleting domain custom resource {0}", domainUid); - assertTrue(deleteDomainCustomResource(domainUid, domainNS)); - - // wait until domain was deleted - withStandardRetryPolicy - .conditionEvaluationListener( - condition -> logger.info("Waiting for domain {0} to be deleted in namespace {1} " - + "(elapsed time {2}ms, remaining time {3}ms)", - domainUid, - domainNS, - condition.getElapsedTimeInMS(), - condition.getRemainingTimeInMS())) - .until(domainDoesNotExist(domainUid, DOMAIN_VERSION, domainNS)); - } - private HelmParams installAndVerifyOperatorCanManageDomainBySelector(Map managedDomains, Map unmanagedDomains, String selector, String selectorValue, diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiCommonMount.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiCommonMount.java index 32c45548d2a..fb4225ce34b 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiCommonMount.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiCommonMount.java @@ -13,6 +13,7 @@ import java.util.Map; import io.kubernetes.client.custom.V1Patch; +import io.kubernetes.client.openapi.models.CoreV1Event; import oracle.weblogic.domain.CommonMount; import oracle.weblogic.domain.Domain; import oracle.weblogic.kubernetes.actions.impl.primitive.Command; @@ -44,24 +45,30 @@ import static oracle.weblogic.kubernetes.actions.ActionConstants.MODEL_DIR; import static oracle.weblogic.kubernetes.actions.ActionConstants.RESOURCE_DIR; import static oracle.weblogic.kubernetes.actions.TestActions.buildAppArchive; +import static oracle.weblogic.kubernetes.actions.TestActions.createDomainCustomResource; import static oracle.weblogic.kubernetes.actions.TestActions.defaultAppParams; import static oracle.weblogic.kubernetes.actions.TestActions.deleteImage; import static oracle.weblogic.kubernetes.actions.TestActions.dockerPush; import static oracle.weblogic.kubernetes.actions.TestActions.getDomainCustomResource; import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort; +import static oracle.weblogic.kubernetes.actions.TestActions.now; import static oracle.weblogic.kubernetes.actions.TestActions.patchDomainCustomResource; import static oracle.weblogic.kubernetes.assertions.TestAssertions.verifyRollingRestartOccurred; import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createDomainResource; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkEvent; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkSystemResourceConfig; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkSystemResourceConfiguration; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDomainAndVerify; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createOcirRepoSecret; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createSecretWithUsernamePassword; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.deleteDomainResource; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getExternalServicePodName; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getPodsWithTimeStamps; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyOperator; import static oracle.weblogic.kubernetes.utils.FileUtils.replaceStringInFile; import static oracle.weblogic.kubernetes.utils.FileUtils.unzipWDTInstallationFile; +import static oracle.weblogic.kubernetes.utils.K8sEvents.DOMAIN_PROCESSING_FAILED; +import static oracle.weblogic.kubernetes.utils.K8sEvents.getEvent; import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; import static org.awaitility.Awaitility.with; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @@ -76,6 +83,7 @@ public class ItMiiCommonMount { private static String opNamespace = null; private static String domainNamespace = null; + private static String errorpathDomainNamespace = null; private static LoggingFacade logger = null; private String domainUid = "domain1"; private static String miiCMImage1 = MII_COMMONMOUNT_IMAGE_NAME + ":" + MII_BASIC_IMAGE_TAG + "1"; @@ -97,7 +105,7 @@ public class ItMiiCommonMount { * JUnit engine parameter resolution mechanism */ @BeforeAll - public static void initAll(@Namespaces(2) List namespaces) { + public static void initAll(@Namespaces(3) List namespaces) { logger = getLogger(); // get a new unique opNamespace logger.info("Creating unique namespace for Operator"); @@ -108,8 +116,12 @@ public static void initAll(@Namespaces(2) List namespaces) { assertNotNull(namespaces.get(1), "Namespace list is null"); domainNamespace = namespaces.get(1); + logger.info("Creating unique namespace for errorpathDomain"); + assertNotNull(namespaces.get(2), "Namespace list is null"); + errorpathDomainNamespace = namespaces.get(2); + // install and verify operator - installAndVerifyOperator(opNamespace, domainNamespace); + installAndVerifyOperator(opNamespace, domainNamespace, errorpathDomainNamespace); } @@ -282,6 +294,260 @@ public void testUpdateDataSourceInDomainUsingCommonMount1() { logger.info("Found the DataResource configuration"); } + /** + * Negative Test to create domain with mismatch common mount path in common mount image and in commonMountVolumes. + * in commonMountVolumes, set mountPath to "/errorpath" + * in common mount image, set COMMON_MOUNT_PATH to "/common" + */ + @Test + @Order(3) + @DisplayName("Negative Test to create domain with mismatch common mount path in common mount image and " + + "in commonMountVolumes") + public void testErrorPathDomainMismatchCommonDirectory() { + + OffsetDateTime timestamp = now(); + String errorPathCMImage1 = MII_COMMONMOUNT_IMAGE_NAME + ":errorpathimage1"; + + // admin/managed server name here should match with model yaml + final String commonMountVolumeName = "commonMountsVolume1"; + final String commonMountPath = "/errorpath"; + + // Create the repo secret to pull the image + // this secret is used only for non-kind cluster + createOcirRepoSecret(errorpathDomainNamespace); + + // create secret for admin credentials + logger.info("Create secret for admin credentials"); + String adminSecretName = "weblogic-credentials"; + createSecretWithUsernamePassword(adminSecretName, errorpathDomainNamespace, + ADMIN_USERNAME_DEFAULT, ADMIN_PASSWORD_DEFAULT); + + // create encryption secret + logger.info("Create encryption secret"); + String encryptionSecretName = "encryptionsecret"; + createSecretWithUsernamePassword(encryptionSecretName, errorpathDomainNamespace, + "weblogicenc", "weblogicenc"); + + // create stage dir for common mount image + Path errorpathCMPath1 = Paths.get(RESULTS_ROOT, "errorpathcmimage1"); + assertDoesNotThrow(() -> FileUtils.deleteDirectory(errorpathCMPath1.toFile())); + assertDoesNotThrow(() -> Files.createDirectories(errorpathCMPath1)); + + // create models dir and copy model, archive files if any for image1 + Path modelsPath1 = Paths.get(errorpathCMPath1.toString(), "models"); + assertDoesNotThrow(() -> Files.createDirectories(modelsPath1)); + assertDoesNotThrow(() -> Files.copy( + Paths.get(MODEL_DIR, MII_BASIC_WDT_MODEL_FILE), + Paths.get(modelsPath1.toString(), MII_BASIC_WDT_MODEL_FILE), + StandardCopyOption.REPLACE_EXISTING)); + + // unzip WDT installation file into work dir + unzipWDTInstallationFile(errorpathCMPath1.toString()); + + // create image1 with model and wdt installation files + createCommonMountImage(errorpathCMPath1.toString(), + Paths.get(RESOURCE_DIR, "commonmount", "Dockerfile").toString(), errorPathCMImage1); + + // push image1 to repo for multi node cluster + if (!DOMAIN_IMAGES_REPO.isEmpty()) { + logger.info("docker push image {0} to registry {1}", errorPathCMImage1, DOMAIN_IMAGES_REPO); + assertTrue(dockerPush(errorPathCMImage1), String.format("docker push failed for image %s", errorPathCMImage1)); + } + + // create domain custom resource using common mount and images + logger.info("Creating domain custom resource with domainUid {0} and common mount image {1}", + domainUid, errorPathCMImage1); + Domain domainCR = createDomainResource(domainUid, errorpathDomainNamespace, + WEBLOGIC_IMAGE_NAME + ":" + WEBLOGIC_IMAGE_TAG, adminSecretName, OCIR_SECRET_NAME, + encryptionSecretName, replicaCount, "cluster-1", commonMountPath, + commonMountVolumeName, errorPathCMImage1); + + // create domain and verify it is failed + logger.info("Creating domain {0} with common mount image {1} in namespace {2}", + domainUid, errorPathCMImage1, errorpathDomainNamespace); + + assertDoesNotThrow(() -> createDomainCustomResource(domainCR), + "createDomainCustomResource throws Exception"); + + // check the domain event contains the expected error msg + checkEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); + CoreV1Event event = + getEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); + String expectedErrorMsg = "Failed to complete processing domain resource domain1 due to: " + + "Common Mount: Dir '/errorpath' doesn't exist or is empty. Exiting."; + assertTrue(event.getMessage().contains(expectedErrorMsg), + String.format("The event message does not contain the expected error msg %s", expectedErrorMsg)); + + // delete domain1 + deleteDomainResource(errorpathDomainNamespace, domainUid); + } + + /** + * Negative Test to create domain without WDT binary. + */ + @Test + @Order(4) + @DisplayName("Negative Test to create domain without WDT binary") + public void testErrorPathDomainMissingWDTBinary() { + + OffsetDateTime timestamp = now(); + String errorPathCMImage2 = MII_COMMONMOUNT_IMAGE_NAME + ":errorpathimage2"; + + // admin/managed server name here should match with model yaml + final String commonMountVolumeName = "commonMountsVolume1"; + final String commonMountPath = "/common"; + + // Create the repo secret to pull the image + // this secret is used only for non-kind cluster + createOcirRepoSecret(errorpathDomainNamespace); + + // create secret for admin credentials + logger.info("Create secret for admin credentials"); + String adminSecretName = "weblogic-credentials"; + createSecretWithUsernamePassword(adminSecretName, errorpathDomainNamespace, + ADMIN_USERNAME_DEFAULT, ADMIN_PASSWORD_DEFAULT); + + // create encryption secret + logger.info("Create encryption secret"); + String encryptionSecretName = "encryptionsecret"; + createSecretWithUsernamePassword(encryptionSecretName, errorpathDomainNamespace, + "weblogicenc", "weblogicenc"); + + // create stage dir for common mount image + Path errorpathCMPath2 = Paths.get(RESULTS_ROOT, "errorpathcmimage2"); + assertDoesNotThrow(() -> FileUtils.deleteDirectory(errorpathCMPath2.toFile())); + assertDoesNotThrow(() -> Files.createDirectories(errorpathCMPath2)); + + // create models dir and copy model, archive files if any for image1 + Path modelsPath2 = Paths.get(errorpathCMPath2.toString(), "models"); + assertDoesNotThrow(() -> Files.createDirectories(modelsPath2)); + assertDoesNotThrow(() -> Files.copy( + Paths.get(MODEL_DIR, MII_BASIC_WDT_MODEL_FILE), + Paths.get(modelsPath2.toString(), MII_BASIC_WDT_MODEL_FILE), + StandardCopyOption.REPLACE_EXISTING)); + + // create image1 with model and wdt installation files + createCommonMountImage(errorpathCMPath2.toString(), + Paths.get(RESOURCE_DIR, "commonmount", "Dockerfile").toString(), errorPathCMImage2); + + // push image1 to repo for multi node cluster + if (!DOMAIN_IMAGES_REPO.isEmpty()) { + logger.info("docker push image {0} to registry {1}", errorPathCMImage2, DOMAIN_IMAGES_REPO); + assertTrue(dockerPush(errorPathCMImage2), String.format("docker push failed for image %s", errorPathCMImage2)); + } + + // create domain custom resource using common mount and images + logger.info("Creating domain custom resource with domainUid {0} and common mount image {1}", + domainUid, errorPathCMImage2); + Domain domainCR = createDomainResource(domainUid, errorpathDomainNamespace, + WEBLOGIC_IMAGE_NAME + ":" + WEBLOGIC_IMAGE_TAG, adminSecretName, OCIR_SECRET_NAME, + encryptionSecretName, replicaCount, "cluster-1", commonMountPath, + commonMountVolumeName, errorPathCMImage2); + + // create domain and verify it is failed + logger.info("Creating domain {0} with common mount image {1} in namespace {2}", + domainUid, errorPathCMImage2, errorpathDomainNamespace); + + assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); + + // check the domain event contains the expected error msg + checkEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); + CoreV1Event event = + getEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); + String expectedErrorMsg = "The domain resource 'spec.domainHomeSourceType' is 'FromModel' and " + + "a WebLogic Deploy Tool (WDT) install is not located at 'spec.configuration.model.wdtInstallHome' " + + "which is currently set to '/common/weblogic-deploy'"; + assertTrue(event.getMessage().contains(expectedErrorMsg), + String.format("The event message does not contain the expected error msg %s", expectedErrorMsg)); + + // delete domain1 + deleteDomainResource(errorpathDomainNamespace, domainUid); + } + + /** + * Negative Test to create domain without domain model file, the common mount contains only sparse JMS config. + */ + @Test + @Order(5) + @DisplayName("Negative Test to create domain without domain model file, only having sparse JMS config") + public void testErrorPathDomainMissingDomainConfig() { + + OffsetDateTime timestamp = now(); + String errorPathCMImage3 = MII_COMMONMOUNT_IMAGE_NAME + ":errorpathimage3"; + + // admin/managed server name here should match with model yaml + final String commonMountVolumeName = "commonMountsVolume1"; + final String commonMountPath = "/common"; + + // Create the repo secret to pull the image + // this secret is used only for non-kind cluster + createOcirRepoSecret(errorpathDomainNamespace); + + // create secret for admin credentials + logger.info("Create secret for admin credentials"); + String adminSecretName = "weblogic-credentials"; + createSecretWithUsernamePassword(adminSecretName, errorpathDomainNamespace, + ADMIN_USERNAME_DEFAULT, ADMIN_PASSWORD_DEFAULT); + + // create encryption secret + logger.info("Create encryption secret"); + String encryptionSecretName = "encryptionsecret"; + createSecretWithUsernamePassword(encryptionSecretName, errorpathDomainNamespace, + "weblogicenc", "weblogicenc"); + + // create stage dir for common mount image + Path errorpathCMPath3 = Paths.get(RESULTS_ROOT, "errorpathcmimage3"); + assertDoesNotThrow(() -> FileUtils.deleteDirectory(errorpathCMPath3.toFile())); + assertDoesNotThrow(() -> Files.createDirectories(errorpathCMPath3)); + + // create models dir and copy model, archive files if any for image + Path modelsPath3 = Paths.get(errorpathCMPath3.toString(), "models"); + assertDoesNotThrow(() -> Files.createDirectories(modelsPath3)); + assertDoesNotThrow(() -> Files.copy( + Paths.get(MODEL_DIR, "model.jms2.yaml"), + Paths.get(modelsPath3.toString(), "model.jms2.yaml"), + StandardCopyOption.REPLACE_EXISTING)); + + // unzip WDT installation file into work dir + unzipWDTInstallationFile(errorpathCMPath3.toString()); + + // create image1 with model and wdt installation files + createCommonMountImage(errorpathCMPath3.toString(), + Paths.get(RESOURCE_DIR, "commonmount", "Dockerfile").toString(), errorPathCMImage3); + + // push image1 to repo for multi node cluster + if (!DOMAIN_IMAGES_REPO.isEmpty()) { + logger.info("docker push image {0} to registry {1}", errorPathCMImage3, DOMAIN_IMAGES_REPO); + assertTrue(dockerPush(errorPathCMImage3), String.format("docker push failed for image %s", errorPathCMImage3)); + } + + // create domain custom resource using common mount and images + logger.info("Creating domain custom resource with domainUid {0} and common mount image {1}", + domainUid, errorPathCMImage3); + Domain domainCR = createDomainResource(domainUid, errorpathDomainNamespace, + WEBLOGIC_IMAGE_NAME + ":" + WEBLOGIC_IMAGE_TAG, adminSecretName, OCIR_SECRET_NAME, + encryptionSecretName, replicaCount, "cluster-1", commonMountPath, + commonMountVolumeName, errorPathCMImage3); + + // create domain and verify it is failed + logger.info("Creating domain {0} with common mount image {1} in namespace {2}", + domainUid, errorPathCMImage3, errorpathDomainNamespace); + + assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); + + // check the domain event contains the expected error msg + checkEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); + CoreV1Event event = + getEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); + String expectedErrorMsg = + "createDomain did not find the required domainInfo section in the model file /common/models/model.jms2.yaml"; + assertTrue(event.getMessage().contains(expectedErrorMsg), + String.format("The event message does not contain the expected error msg %s", expectedErrorMsg)); + + // delete domain1 + deleteDomainResource(errorpathDomainNamespace, domainUid); + } + private static void patchDomainWithCMImageAndVerify(String oldImageName, String newImageName, String domainUid, String domainNamespace) { String adminServerPodName = domainUid + "-admin-server"; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java index d5386855dc8..f26b4f3a0e3 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java @@ -395,6 +395,32 @@ public static Callable domainStatusReasonMatches(oracle.weblogic.domain }; } + /** + * Check the status message of the domain matches the given message. + * @param domain oracle.weblogic.domain.Domain object + * @param statusMessage the expected status message of the domain + * @return true if the status message matches, false otherwise + */ + public static Callable domainStatusMessageMatches(oracle.weblogic.domain.Domain domain, + String statusMessage) { + LoggingFacade logger = getLogger(); + return () -> { + if (domain != null && domain.getStatus() != null && domain.getStatus().getReason() != null) { + logger.info("domain status reason: {0}", domain.getStatus().getReason()); + return domain.getStatus().getReason().equalsIgnoreCase(statusMessage); + } else { + if (domain == null) { + logger.info("domain is null"); + } else if (domain.getStatus() == null) { + logger.info("domain status is null"); + } else { + logger.info("domain status reason is null"); + } + return false; + } + }; + } + /** * Check if a loadbalancer pod is ready. * diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java index da3ae407e47..92b91f0d578 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java @@ -185,6 +185,7 @@ import static oracle.weblogic.kubernetes.actions.TestActions.createService; import static oracle.weblogic.kubernetes.actions.TestActions.createServiceAccount; import static oracle.weblogic.kubernetes.actions.TestActions.defaultAppParams; +import static oracle.weblogic.kubernetes.actions.TestActions.deleteDomainCustomResource; import static oracle.weblogic.kubernetes.actions.TestActions.dockerLogin; import static oracle.weblogic.kubernetes.actions.TestActions.dockerPush; import static oracle.weblogic.kubernetes.actions.TestActions.getJob; @@ -220,7 +221,9 @@ import static oracle.weblogic.kubernetes.assertions.TestAssertions.credentialsNotValid; import static oracle.weblogic.kubernetes.assertions.TestAssertions.credentialsValid; import static oracle.weblogic.kubernetes.assertions.TestAssertions.doesImageExist; +import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainDoesNotExist; import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainExists; +import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainStatusMessageMatches; import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainStatusReasonMatches; import static oracle.weblogic.kubernetes.assertions.TestAssertions.isApacheReady; import static oracle.weblogic.kubernetes.assertions.TestAssertions.isElkStackPodReady; @@ -247,6 +250,7 @@ import static oracle.weblogic.kubernetes.assertions.TestAssertions.serviceExists; import static oracle.weblogic.kubernetes.utils.ExecCommand.exec; import static oracle.weblogic.kubernetes.utils.FileUtils.checkDirectory; +import static oracle.weblogic.kubernetes.utils.K8sEvents.checkDomainEvent; import static oracle.weblogic.kubernetes.utils.TestUtils.callWebAppAndCheckForServerNameInResponse; import static oracle.weblogic.kubernetes.utils.TestUtils.callWebAppAndWaitTillReady; import static oracle.weblogic.kubernetes.utils.TestUtils.getNextFreePort; @@ -2034,6 +2038,27 @@ public static void checkDomainStatusReasonMatches(Domain domain, String namespac .until(assertDoesNotThrow(() -> domainStatusReasonMatches(domain, statusReason))); } + /** + * Check the status message of the domain matches the given message. + * + * @param domain oracle.weblogic.domain.Domain object + * @param namespace the namespace in which the domain exists + * @param statusMessage the expected status message of the domain + */ + public static void checkDomainStatusMessageMatches(Domain domain, String namespace, String statusMessage) { + LoggingFacade logger = getLogger(); + withStandardRetryPolicy + .conditionEvaluationListener( + condition -> logger.info("Waiting for the status message of the domain {0} in namespace {1} " + + "is {2} (elapsed time {3}ms, remaining time {4}ms)", + domain, + namespace, + statusMessage, + condition.getElapsedTimeInMS(), + condition.getRemainingTimeInMS())) + .until(assertDoesNotThrow(() -> domainStatusMessageMatches(domain, statusMessage))); + } + /** * Check whether the cluster's replica count matches with input parameter value. * @@ -4327,4 +4352,50 @@ public static String getDockerExtraArgs() { } return extraArgs.toString(); } + + /** + * Wait until a given event is logged by the operator. + * + * @param opNamespace namespace in which the operator is running + * @param domainNamespace namespace in which the domain exists + * @param domainUid UID of the domain + * @param reason event to check for Created, Changed, deleted, processing etc + * @param type type of event, Normal of Warning + * @param timestamp the timestamp after which to see events + */ + public static void checkEvent( + String opNamespace, String domainNamespace, String domainUid, + String reason, String type, OffsetDateTime timestamp) { + withStandardRetryPolicy + .conditionEvaluationListener(condition -> + getLogger().info("Waiting for domain event {0} to be logged in namespace {1} " + + "(elapsed time {2}ms, remaining time {3}ms)", + reason, + domainNamespace, + condition.getElapsedTimeInMS(), + condition.getRemainingTimeInMS())) + .until(checkDomainEvent(opNamespace, domainNamespace, domainUid, reason, type, timestamp)); + } + + /** + * Delete a domain in the specified namespace. + * @param domainNS the namespace in which the domain exists + * @param domainUid domain uid + */ + public static void deleteDomainResource(String domainNS, String domainUid) { + //clean up domain resources in namespace and set namespace to label , managed by operator + getLogger().info("deleting domain custom resource {0}", domainUid); + assertTrue(deleteDomainCustomResource(domainUid, domainNS)); + + // wait until domain was deleted + withStandardRetryPolicy + .conditionEvaluationListener( + condition -> getLogger().info("Waiting for domain {0} to be deleted in namespace {1} " + + "(elapsed time {2}ms, remaining time {3}ms)", + domainUid, + domainNS, + condition.getElapsedTimeInMS(), + condition.getRemainingTimeInMS())) + .until(domainDoesNotExist(domainUid, DOMAIN_VERSION, domainNS)); + } } diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/K8sEvents.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/K8sEvents.java index 2f22ce3f35f..eb33b62c96b 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/K8sEvents.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/K8sEvents.java @@ -159,7 +159,7 @@ public static boolean domainEventExists( } } } catch (ApiException ex) { - Logger.getLogger(ItKubernetesEvents.class.getName()).log(Level.SEVERE, null, ex); + logger.log(Level.SEVERE, null, ex); } return false; } From 6b8e9eee831d685ba3b0e099f36aeb759dd6cc40 Mon Sep 17 00:00:00 2001 From: xiancao Date: Mon, 12 Jul 2021 21:36:23 +0000 Subject: [PATCH 2/8] change default auxiliary image path in dockerfile --- integration-tests/src/test/resources/auxiliaryimage/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/src/test/resources/auxiliaryimage/Dockerfile b/integration-tests/src/test/resources/auxiliaryimage/Dockerfile index 53adfa77e9e..b0d28852f81 100644 --- a/integration-tests/src/test/resources/auxiliaryimage/Dockerfile +++ b/integration-tests/src/test/resources/auxiliaryimage/Dockerfile @@ -17,7 +17,7 @@ # FROM busybox -ARG AUXILIARY_IMAGE_PATH=/common +ARG AUXILIARY_IMAGE_PATH=/auxiliary ARG USER=oracle ARG USERID=1000 ARG GROUP=root From 8db94e11638c6c71c071776c0d5bffb75d1f7893 Mon Sep 17 00:00:00 2001 From: xiancao Date: Tue, 13 Jul 2021 17:03:04 +0000 Subject: [PATCH 3/8] add more tests for auxiliary image --- .../kubernetes/ItMiiAuxiliaryImage.java | 311 +++++++++++------- .../kubernetes/utils/CommonMiiTestUtils.java | 46 +++ 2 files changed, 232 insertions(+), 125 deletions(-) diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java index a1b471a8fc8..a45543ec34e 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java @@ -8,6 +8,7 @@ import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.time.OffsetDateTime; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -15,6 +16,7 @@ import io.kubernetes.client.custom.V1Patch; import io.kubernetes.client.openapi.models.CoreV1Event; import oracle.weblogic.domain.AuxiliaryImage; +import oracle.weblogic.domain.AuxiliaryImageVolume; import oracle.weblogic.domain.Domain; import oracle.weblogic.kubernetes.actions.impl.primitive.Command; import oracle.weblogic.kubernetes.actions.impl.primitive.CommandParams; @@ -55,6 +57,7 @@ import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort; import static oracle.weblogic.kubernetes.actions.TestActions.now; import static oracle.weblogic.kubernetes.actions.TestActions.patchDomainCustomResource; +import static oracle.weblogic.kubernetes.assertions.TestAssertions.secretExists; import static oracle.weblogic.kubernetes.assertions.TestAssertions.verifyRollingRestartOccurred; import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createDomainResource; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkEvent; @@ -96,6 +99,8 @@ public class ItMiiAuxiliaryImage { private final String adminServerPodName = domainUid + "-admin-server"; private final String managedServerPrefix = domainUid + "-managed-server"; private final int replicaCount = 2; + private String adminSecretName = "weblogic-credentials"; + private String encryptionSecretName = "encryptionsecret"; ConditionFactory withStandardRetryPolicy = with().pollDelay(0, SECONDS) @@ -298,46 +303,30 @@ public void testUpdateDataSourceInDomainUsingAuxiliaryImage() { } /** - * Negative Test to create domain with mismatch common mount path in common mount image and in commonMountVolumes. - * in commonMountVolumes, set mountPath to "/errorpath" - * in common mount image, set COMMON_MOUNT_PATH to "/common" + * Negative Test to create domain with mismatch mount path in auxiliary image and auxiliaryImageVolumes. + * in auxiliaryImageVolumes, set mountPath to "/errorpath" + * in auxiliary image, set AUXILIARY_IMAGE_PATH to "/auxiliary" */ @Test @Order(3) - @DisplayName("Negative Test to create domain with mismatch common mount path in common mount image and " - + "in commonMountVolumes") - public void testErrorPathDomainMismatchCommonDirectory() { + @DisplayName("Negative Test to create domain with mismatch mount path in auxiliary image and auxiliaryImageVolumes") + public void testErrorPathDomainMismatchMountPath() { OffsetDateTime timestamp = now(); - String errorPathCMImage1 = MII_AUXILIARY_IMAGE_NAME + ":errorpathimage1"; + String errorPathAuxiliaryImage1 = MII_AUXILIARY_IMAGE_NAME + ":errorpathimage1"; - // admin/managed server name here should match with model yaml - final String commonMountVolumeName = "commonMountsVolume1"; - final String commonMountPath = "/errorpath"; - - // Create the repo secret to pull the image - // this secret is used only for non-kind cluster - createOcirRepoSecret(errorpathDomainNamespace); - - // create secret for admin credentials - logger.info("Create secret for admin credentials"); - String adminSecretName = "weblogic-credentials"; - createSecretWithUsernamePassword(adminSecretName, errorpathDomainNamespace, - ADMIN_USERNAME_DEFAULT, ADMIN_PASSWORD_DEFAULT); + final String auxiliaryImageVolumeName = "auxiliaryImageVolume1"; + final String auxiliaryImagePath = "/errorpath"; - // create encryption secret - logger.info("Create encryption secret"); - String encryptionSecretName = "encryptionsecret"; - createSecretWithUsernamePassword(encryptionSecretName, errorpathDomainNamespace, - "weblogicenc", "weblogicenc"); + createSecretsForDomain(adminSecretName, encryptionSecretName, errorpathDomainNamespace); - // create stage dir for common mount image - Path errorpathCMPath1 = Paths.get(RESULTS_ROOT, "errorpathcmimage1"); - assertDoesNotThrow(() -> FileUtils.deleteDirectory(errorpathCMPath1.toFile())); - assertDoesNotThrow(() -> Files.createDirectories(errorpathCMPath1)); + // create stage dir for auxiliary image + Path errorpathAIPath1 = Paths.get(RESULTS_ROOT, "errorpathauxiimage1"); + assertDoesNotThrow(() -> FileUtils.deleteDirectory(errorpathAIPath1.toFile())); + assertDoesNotThrow(() -> Files.createDirectories(errorpathAIPath1)); - // create models dir and copy model, archive files if any for image1 - Path modelsPath1 = Paths.get(errorpathCMPath1.toString(), "models"); + // create models dir and copy model for image + Path modelsPath1 = Paths.get(errorpathAIPath1.toString(), "models"); assertDoesNotThrow(() -> Files.createDirectories(modelsPath1)); assertDoesNotThrow(() -> Files.copy( Paths.get(MODEL_DIR, MII_BASIC_WDT_MODEL_FILE), @@ -345,39 +334,39 @@ public void testErrorPathDomainMismatchCommonDirectory() { StandardCopyOption.REPLACE_EXISTING)); // unzip WDT installation file into work dir - unzipWDTInstallationFile(errorpathCMPath1.toString()); + unzipWDTInstallationFile(errorpathAIPath1.toString()); - // create image1 with model and wdt installation files - createAuxiliaryImage(errorpathCMPath1.toString(), - Paths.get(RESOURCE_DIR, "commonmount", "Dockerfile").toString(), errorPathCMImage1); + // create image with model and wdt installation files + createAuxiliaryImage(errorpathAIPath1.toString(), + Paths.get(RESOURCE_DIR, "auxiliaryimage", "Dockerfile").toString(), errorPathAuxiliaryImage1); - // push image1 to repo for multi node cluster + // push image to repo for multi node cluster if (!DOMAIN_IMAGES_REPO.isEmpty()) { - logger.info("docker push image {0} to registry {1}", errorPathCMImage1, DOMAIN_IMAGES_REPO); - assertTrue(dockerPush(errorPathCMImage1), String.format("docker push failed for image %s", errorPathCMImage1)); + logger.info("docker push image {0} to registry {1}", errorPathAuxiliaryImage1, DOMAIN_IMAGES_REPO); + assertTrue(dockerPush(errorPathAuxiliaryImage1), + String.format("docker push failed for image %s", errorPathAuxiliaryImage1)); } - // create domain custom resource using common mount and images - logger.info("Creating domain custom resource with domainUid {0} and common mount image {1}", - domainUid, errorPathCMImage1); + // create domain custom resource using auxiliary images + logger.info("Creating domain custom resource with domainUid {0} and auxiliary image {1}", + domainUid, errorPathAuxiliaryImage1); Domain domainCR = createDomainResource(domainUid, errorpathDomainNamespace, WEBLOGIC_IMAGE_NAME + ":" + WEBLOGIC_IMAGE_TAG, adminSecretName, OCIR_SECRET_NAME, - encryptionSecretName, replicaCount, "cluster-1", commonMountPath, - commonMountVolumeName, errorPathCMImage1); + encryptionSecretName, replicaCount, "cluster-1", auxiliaryImagePath, + auxiliaryImageVolumeName, errorPathAuxiliaryImage1); // create domain and verify it is failed - logger.info("Creating domain {0} with common mount image {1} in namespace {2}", - domainUid, errorPathCMImage1, errorpathDomainNamespace); + logger.info("Creating domain {0} with auxiliary image {1} in namespace {2}", + domainUid, errorPathAuxiliaryImage1, errorpathDomainNamespace); - assertDoesNotThrow(() -> createDomainCustomResource(domainCR), - "createDomainCustomResource throws Exception"); + assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); // check the domain event contains the expected error msg checkEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); CoreV1Event event = getEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); String expectedErrorMsg = "Failed to complete processing domain resource domain1 due to: " - + "Common Mount: Dir '/errorpath' doesn't exist or is empty. Exiting."; + + "Auxiliary Image: Dir '/errorpath' doesn't exist or is empty. Exiting."; assertTrue(event.getMessage().contains(expectedErrorMsg), String.format("The event message does not contain the expected error msg %s", expectedErrorMsg)); @@ -394,62 +383,48 @@ public void testErrorPathDomainMismatchCommonDirectory() { public void testErrorPathDomainMissingWDTBinary() { OffsetDateTime timestamp = now(); - String errorPathCMImage2 = MII_AUXILIARY_IMAGE_NAME + ":errorpathimage2"; - - // admin/managed server name here should match with model yaml - final String commonMountVolumeName = "commonMountsVolume1"; - final String commonMountPath = "/common"; + String errorPathAuxiliaryImage2 = MII_AUXILIARY_IMAGE_NAME + ":errorpathimage2"; - // Create the repo secret to pull the image - // this secret is used only for non-kind cluster - createOcirRepoSecret(errorpathDomainNamespace); - - // create secret for admin credentials - logger.info("Create secret for admin credentials"); - String adminSecretName = "weblogic-credentials"; - createSecretWithUsernamePassword(adminSecretName, errorpathDomainNamespace, - ADMIN_USERNAME_DEFAULT, ADMIN_PASSWORD_DEFAULT); + final String auxiliaryImageVolumeName = "auxiliaryImageVolume1"; + final String auxiliaryImagePath = "/auxiliary"; - // create encryption secret - logger.info("Create encryption secret"); - String encryptionSecretName = "encryptionsecret"; - createSecretWithUsernamePassword(encryptionSecretName, errorpathDomainNamespace, - "weblogicenc", "weblogicenc"); + createSecretsForDomain(adminSecretName, encryptionSecretName, errorpathDomainNamespace); // create stage dir for common mount image - Path errorpathCMPath2 = Paths.get(RESULTS_ROOT, "errorpathcmimage2"); - assertDoesNotThrow(() -> FileUtils.deleteDirectory(errorpathCMPath2.toFile())); - assertDoesNotThrow(() -> Files.createDirectories(errorpathCMPath2)); + Path errorpathAIPath2 = Paths.get(RESULTS_ROOT, "errorpathauxiimage2"); + assertDoesNotThrow(() -> FileUtils.deleteDirectory(errorpathAIPath2.toFile())); + assertDoesNotThrow(() -> Files.createDirectories(errorpathAIPath2)); - // create models dir and copy model, archive files if any for image1 - Path modelsPath2 = Paths.get(errorpathCMPath2.toString(), "models"); + // create models dir and copy model for image + Path modelsPath2 = Paths.get(errorpathAIPath2.toString(), "models"); assertDoesNotThrow(() -> Files.createDirectories(modelsPath2)); assertDoesNotThrow(() -> Files.copy( Paths.get(MODEL_DIR, MII_BASIC_WDT_MODEL_FILE), Paths.get(modelsPath2.toString(), MII_BASIC_WDT_MODEL_FILE), StandardCopyOption.REPLACE_EXISTING)); - // create image1 with model and wdt installation files - createAuxiliaryImage(errorpathCMPath2.toString(), - Paths.get(RESOURCE_DIR, "commonmount", "Dockerfile").toString(), errorPathCMImage2); + // create image with model and no wdt installation files + createAuxiliaryImage(errorpathAIPath2.toString(), + Paths.get(RESOURCE_DIR, "auxiliaryimage", "Dockerfile").toString(), errorPathAuxiliaryImage2); - // push image1 to repo for multi node cluster + // push image to repo for multi node cluster if (!DOMAIN_IMAGES_REPO.isEmpty()) { - logger.info("docker push image {0} to registry {1}", errorPathCMImage2, DOMAIN_IMAGES_REPO); - assertTrue(dockerPush(errorPathCMImage2), String.format("docker push failed for image %s", errorPathCMImage2)); + logger.info("docker push image {0} to registry {1}", errorPathAuxiliaryImage2, DOMAIN_IMAGES_REPO); + assertTrue(dockerPush(errorPathAuxiliaryImage2), + String.format("docker push failed for image %s", errorPathAuxiliaryImage2)); } - // create domain custom resource using common mount and images - logger.info("Creating domain custom resource with domainUid {0} and common mount image {1}", - domainUid, errorPathCMImage2); + // create domain custom resource using auxiliary images + logger.info("Creating domain custom resource with domainUid {0} and auxiliary image {1}", + domainUid, errorPathAuxiliaryImage2); Domain domainCR = createDomainResource(domainUid, errorpathDomainNamespace, WEBLOGIC_IMAGE_NAME + ":" + WEBLOGIC_IMAGE_TAG, adminSecretName, OCIR_SECRET_NAME, - encryptionSecretName, replicaCount, "cluster-1", commonMountPath, - commonMountVolumeName, errorPathCMImage2); + encryptionSecretName, replicaCount, "cluster-1", auxiliaryImagePath, + auxiliaryImageVolumeName, errorPathAuxiliaryImage2); // create domain and verify it is failed logger.info("Creating domain {0} with common mount image {1} in namespace {2}", - domainUid, errorPathCMImage2, errorpathDomainNamespace); + domainUid, errorPathAuxiliaryImage2, errorpathDomainNamespace); assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); @@ -459,7 +434,7 @@ public void testErrorPathDomainMissingWDTBinary() { getEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); String expectedErrorMsg = "The domain resource 'spec.domainHomeSourceType' is 'FromModel' and " + "a WebLogic Deploy Tool (WDT) install is not located at 'spec.configuration.model.wdtInstallHome' " - + "which is currently set to '/common/weblogic-deploy'"; + + "which is currently set to '/auxiliary/weblogic-deploy'"; assertTrue(event.getMessage().contains(expectedErrorMsg), String.format("The event message does not contain the expected error msg %s", expectedErrorMsg)); @@ -468,7 +443,7 @@ public void testErrorPathDomainMissingWDTBinary() { } /** - * Negative Test to create domain without domain model file, the common mount contains only sparse JMS config. + * Negative Test to create domain without domain model file, the auxiliary image contains only sparse JMS config. */ @Test @Order(5) @@ -476,35 +451,20 @@ public void testErrorPathDomainMissingWDTBinary() { public void testErrorPathDomainMissingDomainConfig() { OffsetDateTime timestamp = now(); - String errorPathCMImage3 = MII_AUXILIARY_IMAGE_NAME + ":errorpathimage3"; + String errorPathAuxiliaryImage3 = MII_AUXILIARY_IMAGE_NAME + ":errorpathimage3"; - // admin/managed server name here should match with model yaml - final String commonMountVolumeName = "commonMountsVolume1"; - final String commonMountPath = "/common"; - - // Create the repo secret to pull the image - // this secret is used only for non-kind cluster - createOcirRepoSecret(errorpathDomainNamespace); - - // create secret for admin credentials - logger.info("Create secret for admin credentials"); - String adminSecretName = "weblogic-credentials"; - createSecretWithUsernamePassword(adminSecretName, errorpathDomainNamespace, - ADMIN_USERNAME_DEFAULT, ADMIN_PASSWORD_DEFAULT); + final String auxiliaryImageVolumeName = "auxiliaryImageVolume1"; + final String auxiliaryImagePath = "/auxiliary"; - // create encryption secret - logger.info("Create encryption secret"); - String encryptionSecretName = "encryptionsecret"; - createSecretWithUsernamePassword(encryptionSecretName, errorpathDomainNamespace, - "weblogicenc", "weblogicenc"); + createSecretsForDomain(adminSecretName, encryptionSecretName, errorpathDomainNamespace); - // create stage dir for common mount image - Path errorpathCMPath3 = Paths.get(RESULTS_ROOT, "errorpathcmimage3"); - assertDoesNotThrow(() -> FileUtils.deleteDirectory(errorpathCMPath3.toFile())); - assertDoesNotThrow(() -> Files.createDirectories(errorpathCMPath3)); + // create stage dir for auxiliary image + Path errorpathAIPath3 = Paths.get(RESULTS_ROOT, "errorpathauxiimage3"); + assertDoesNotThrow(() -> FileUtils.deleteDirectory(errorpathAIPath3.toFile())); + assertDoesNotThrow(() -> Files.createDirectories(errorpathAIPath3)); - // create models dir and copy model, archive files if any for image - Path modelsPath3 = Paths.get(errorpathCMPath3.toString(), "models"); + // create models dir and copy model for image + Path modelsPath3 = Paths.get(errorpathAIPath3.toString(), "models"); assertDoesNotThrow(() -> Files.createDirectories(modelsPath3)); assertDoesNotThrow(() -> Files.copy( Paths.get(MODEL_DIR, "model.jms2.yaml"), @@ -512,29 +472,30 @@ public void testErrorPathDomainMissingDomainConfig() { StandardCopyOption.REPLACE_EXISTING)); // unzip WDT installation file into work dir - unzipWDTInstallationFile(errorpathCMPath3.toString()); + unzipWDTInstallationFile(errorpathAIPath3.toString()); // create image1 with model and wdt installation files - createAuxiliaryImage(errorpathCMPath3.toString(), - Paths.get(RESOURCE_DIR, "commonmount", "Dockerfile").toString(), errorPathCMImage3); + createAuxiliaryImage(errorpathAIPath3.toString(), + Paths.get(RESOURCE_DIR, "auxiliaryimage", "Dockerfile").toString(), errorPathAuxiliaryImage3); - // push image1 to repo for multi node cluster + // push image to repo for multi node cluster if (!DOMAIN_IMAGES_REPO.isEmpty()) { - logger.info("docker push image {0} to registry {1}", errorPathCMImage3, DOMAIN_IMAGES_REPO); - assertTrue(dockerPush(errorPathCMImage3), String.format("docker push failed for image %s", errorPathCMImage3)); + logger.info("docker push image {0} to registry {1}", errorPathAuxiliaryImage3, DOMAIN_IMAGES_REPO); + assertTrue(dockerPush(errorPathAuxiliaryImage3), + String.format("docker push failed for image %s", errorPathAuxiliaryImage3)); } - // create domain custom resource using common mount and images - logger.info("Creating domain custom resource with domainUid {0} and common mount image {1}", - domainUid, errorPathCMImage3); + // create domain custom resource using auxiliary images + logger.info("Creating domain custom resource with domainUid {0} and auxiliary image {1}", + domainUid, errorPathAuxiliaryImage3); Domain domainCR = createDomainResource(domainUid, errorpathDomainNamespace, WEBLOGIC_IMAGE_NAME + ":" + WEBLOGIC_IMAGE_TAG, adminSecretName, OCIR_SECRET_NAME, - encryptionSecretName, replicaCount, "cluster-1", commonMountPath, - commonMountVolumeName, errorPathCMImage3); + encryptionSecretName, replicaCount, "cluster-1", auxiliaryImagePath, + auxiliaryImageVolumeName, errorPathAuxiliaryImage3); // create domain and verify it is failed - logger.info("Creating domain {0} with common mount image {1} in namespace {2}", - domainUid, errorPathCMImage3, errorpathDomainNamespace); + logger.info("Creating domain {0} with auxiliary image {1} in namespace {2}", + domainUid, errorPathAuxiliaryImage3, errorpathDomainNamespace); assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); @@ -543,7 +504,88 @@ public void testErrorPathDomainMissingDomainConfig() { CoreV1Event event = getEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); String expectedErrorMsg = - "createDomain did not find the required domainInfo section in the model file /common/models/model.jms2.yaml"; + "createDomain did not find the required domainInfo section in the model file /auxiliary/models/model.jms2.yaml"; + assertTrue(event.getMessage().contains(expectedErrorMsg), + String.format("The event message does not contain the expected error msg %s", expectedErrorMsg)); + + // delete domain1 + deleteDomainResource(errorpathDomainNamespace, domainUid); + } + + /** + * Negative Test to create domain using a custom mount command that's guaranteed to fail, for example: exit 1. + */ + @Test + @Order(6) + @DisplayName("Negative Test to create domain using a custom mount command that's guaranteed to fail") + public void testErrorPathDomainWithFailCustomMountCommand() { + + OffsetDateTime timestamp = now(); + String errorPathAuxiliaryImage4 = MII_AUXILIARY_IMAGE_NAME + ":errorpathimage4"; + + final String auxiliaryImageVolumeName = "auxiliaryImageVolume1"; + final String auxiliaryImagePath = "/auxiliary"; + + createSecretsForDomain(adminSecretName, encryptionSecretName, errorpathDomainNamespace); + + // create stage dir for auxiliary image + Path errorpathAIPath4 = Paths.get(RESULTS_ROOT, "errorpathauxiimage4"); + assertDoesNotThrow(() -> FileUtils.deleteDirectory(errorpathAIPath4.toFile())); + assertDoesNotThrow(() -> Files.createDirectories(errorpathAIPath4)); + + // create models dir and copy model for image + Path modelsPath4 = Paths.get(errorpathAIPath4.toString(), "models"); + assertDoesNotThrow(() -> Files.createDirectories(modelsPath4)); + assertDoesNotThrow(() -> Files.copy( + Paths.get(MODEL_DIR, MII_BASIC_WDT_MODEL_FILE), + Paths.get(modelsPath4.toString(), MII_BASIC_WDT_MODEL_FILE), + StandardCopyOption.REPLACE_EXISTING)); + + // unzip WDT installation file into work dir + unzipWDTInstallationFile(errorpathAIPath4.toString()); + + // create image1 with model and wdt installation files + createAuxiliaryImage(errorpathAIPath4.toString(), + Paths.get(RESOURCE_DIR, "auxiliaryimage", "Dockerfile").toString(), errorPathAuxiliaryImage4); + + // push image to repo for multi node cluster + if (!DOMAIN_IMAGES_REPO.isEmpty()) { + logger.info("docker push image {0} to registry {1}", errorPathAuxiliaryImage4, DOMAIN_IMAGES_REPO); + assertTrue(dockerPush(errorPathAuxiliaryImage4), + String.format("docker push failed for image %s", errorPathAuxiliaryImage4)); + } + + // create domain custom resource using common mount and images + logger.info("Creating domain custom resource with domainUid {0} and auxiliary image {1}", + domainUid, errorPathAuxiliaryImage4); + AuxiliaryImageVolume auxiliaryImageVolume = new AuxiliaryImageVolume() + .name(auxiliaryImageVolumeName) + .mountPath(auxiliaryImagePath); + + AuxiliaryImage auxiliaryImage = new AuxiliaryImage() + .image(errorPathAuxiliaryImage4) + .imagePullPolicy("IfNotPresent") + .volume(auxiliaryImageVolumeName) + .command("exit 1"); + + Domain domainCR = createDomainResource(domainUid, errorpathDomainNamespace, + WEBLOGIC_IMAGE_NAME + ":" + WEBLOGIC_IMAGE_TAG, adminSecretName, OCIR_SECRET_NAME, + encryptionSecretName, replicaCount, "cluster-1", Arrays.asList(auxiliaryImageVolume), + Arrays.asList(auxiliaryImage)); + + // create domain and verify it is failed + logger.info("Creating domain {0} with auxiliary image {1} in namespace {2}", + domainUid, errorPathAuxiliaryImage4, errorpathDomainNamespace); + + assertDoesNotThrow(() -> createDomainCustomResource(domainCR), + "createDomainCustomResource throws Exception"); + + // check the domain event contains the expected error msg + checkEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); + CoreV1Event event = + getEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); + String expectedErrorMsg = "Failed to complete processing domain resource domain1 due to: " + + "Auxiliary Image: Command 'exit 1' execution failed in container"; assertTrue(event.getMessage().contains(expectedErrorMsg), String.format("The event message does not contain the expected error msg %s", expectedErrorMsg)); @@ -644,4 +686,23 @@ private void createAuxiliaryImage(String stageDirPath, String dockerFileLocation .command(cmdToExecute)) .execute(), String.format("Failed to execute", cmdToExecute)); } + + private void createSecretsForDomain(String adminSecretName, String encryptionSecretName, String domainNamespace) { + if (!secretExists(OCIR_SECRET_NAME, domainNamespace)) { + createOcirRepoSecret(domainNamespace); + } + + // create secret for admin credentials + logger.info("Create secret for admin credentials"); + if (!secretExists(adminSecretName, domainNamespace)) { + createSecretWithUsernamePassword(adminSecretName, domainNamespace, + ADMIN_USERNAME_DEFAULT, ADMIN_PASSWORD_DEFAULT); + } + + // create encryption secret + logger.info("Create encryption secret"); + if (!secretExists(encryptionSecretName, domainNamespace)) { + createSecretWithUsernamePassword(encryptionSecretName, domainNamespace, "weblogicenc", "weblogicenc"); + } + } } diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java index 9e06b556a99..4e6d29172c9 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java @@ -299,6 +299,52 @@ public static Domain createDomainResource( return domainCR; } + /** + * Create a domain object for a Kubernetes domain custom resource using the basic WLS image and MII auxiliary image + * image. + * + * @param domainResourceName name of the domain resource + * @param domNamespace Kubernetes namespace that the domain is hosted + * @param baseImageName name of the base image to use + * @param adminSecretName name of the new WebLogic admin credentials secret + * @param repoSecretName name of the secret for pulling the WebLogic image + * @param encryptionSecretName name of the secret used to encrypt the models + * @param replicaCount number of managed servers to start + * @param clusterName name of the cluster to add in domain + * @param auxiliaryImageVolumes list of AuxiliaryImageVolumes + * @param auxiliaryImages list of AuxiliaryImages + * @return domain object of the domain resource + */ + public static Domain createDomainResource( + String domainResourceName, + String domNamespace, + String baseImageName, + String adminSecretName, + String repoSecretName, + String encryptionSecretName, + int replicaCount, + String clusterName, + List auxiliaryImageVolumes, + List auxiliaryImages) { + + Domain domainCR = CommonMiiTestUtils.createDomainResource(domainResourceName, domNamespace, + baseImageName, adminSecretName, repoSecretName, + encryptionSecretName, replicaCount, clusterName); + + for (AuxiliaryImageVolume auxiliaryImageVolume : auxiliaryImageVolumes) { + domainCR.spec().addAuxiliaryImageVolumesItem(auxiliaryImageVolume); + domainCR.spec().configuration().model() + .withModelHome(auxiliaryImageVolume.getMountPath() + "/models") + .withWdtInstallHome(auxiliaryImageVolume.getMountPath() + "/weblogic-deploy"); + } + + for (AuxiliaryImage auxiliaryImage : auxiliaryImages) { + domainCR.spec().serverPod().addAuxiliaryImagesItem(auxiliaryImage); + } + + return domainCR; + } + /** * Create a domain object for a Kubernetes domain custom resource using the basic model-in-image * image. From d93de90b3059aba4dee9ee6862848b38d826986d Mon Sep 17 00:00:00 2001 From: xiancao Date: Tue, 13 Jul 2021 18:03:15 +0000 Subject: [PATCH 4/8] cleanup --- .../kubernetes/ItMiiAuxiliaryImage.java | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java index a45543ec34e..610e2b2e6be 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java @@ -362,13 +362,9 @@ public void testErrorPathDomainMismatchMountPath() { assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); // check the domain event contains the expected error msg - checkEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); - CoreV1Event event = - getEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); String expectedErrorMsg = "Failed to complete processing domain resource domain1 due to: " + "Auxiliary Image: Dir '/errorpath' doesn't exist or is empty. Exiting."; - assertTrue(event.getMessage().contains(expectedErrorMsg), - String.format("The event message does not contain the expected error msg %s", expectedErrorMsg)); + verifyDomainEventContainsExpectedErrorMsg(timestamp, expectedErrorMsg); // delete domain1 deleteDomainResource(errorpathDomainNamespace, domainUid); @@ -429,14 +425,10 @@ public void testErrorPathDomainMissingWDTBinary() { assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); // check the domain event contains the expected error msg - checkEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); - CoreV1Event event = - getEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); String expectedErrorMsg = "The domain resource 'spec.domainHomeSourceType' is 'FromModel' and " + "a WebLogic Deploy Tool (WDT) install is not located at 'spec.configuration.model.wdtInstallHome' " + "which is currently set to '/auxiliary/weblogic-deploy'"; - assertTrue(event.getMessage().contains(expectedErrorMsg), - String.format("The event message does not contain the expected error msg %s", expectedErrorMsg)); + verifyDomainEventContainsExpectedErrorMsg(timestamp, expectedErrorMsg); // delete domain1 deleteDomainResource(errorpathDomainNamespace, domainUid); @@ -500,13 +492,9 @@ public void testErrorPathDomainMissingDomainConfig() { assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); // check the domain event contains the expected error msg - checkEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); - CoreV1Event event = - getEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); String expectedErrorMsg = "createDomain did not find the required domainInfo section in the model file /auxiliary/models/model.jms2.yaml"; - assertTrue(event.getMessage().contains(expectedErrorMsg), - String.format("The event message does not contain the expected error msg %s", expectedErrorMsg)); + verifyDomainEventContainsExpectedErrorMsg(timestamp, expectedErrorMsg); // delete domain1 deleteDomainResource(errorpathDomainNamespace, domainUid); @@ -527,7 +515,7 @@ public void testErrorPathDomainWithFailCustomMountCommand() { final String auxiliaryImagePath = "/auxiliary"; createSecretsForDomain(adminSecretName, encryptionSecretName, errorpathDomainNamespace); - + // create stage dir for auxiliary image Path errorpathAIPath4 = Paths.get(RESULTS_ROOT, "errorpathauxiimage4"); assertDoesNotThrow(() -> FileUtils.deleteDirectory(errorpathAIPath4.toFile())); @@ -581,13 +569,9 @@ public void testErrorPathDomainWithFailCustomMountCommand() { "createDomainCustomResource throws Exception"); // check the domain event contains the expected error msg - checkEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); - CoreV1Event event = - getEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); String expectedErrorMsg = "Failed to complete processing domain resource domain1 due to: " + "Auxiliary Image: Command 'exit 1' execution failed in container"; - assertTrue(event.getMessage().contains(expectedErrorMsg), - String.format("The event message does not contain the expected error msg %s", expectedErrorMsg)); + verifyDomainEventContainsExpectedErrorMsg(timestamp, expectedErrorMsg); // delete domain1 deleteDomainResource(errorpathDomainNamespace, domainUid); @@ -705,4 +689,12 @@ private void createSecretsForDomain(String adminSecretName, String encryptionSec createSecretWithUsernamePassword(encryptionSecretName, domainNamespace, "weblogicenc", "weblogicenc"); } } + + private void verifyDomainEventContainsExpectedErrorMsg(OffsetDateTime timestamp, String expectedErrorMsg) { + checkEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); + CoreV1Event event = + getEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); + assertTrue(event.getMessage().contains(expectedErrorMsg), + String.format("The event message does not contain the expected error msg %s", expectedErrorMsg)); + } } From 13495663f48b7efa8212f74099bbccca905178ef Mon Sep 17 00:00:00 2001 From: xiancao Date: Tue, 13 Jul 2021 18:37:45 +0000 Subject: [PATCH 5/8] cleanup --- .../kubernetes/ItMiiAuxiliaryImage.java | 4 --- .../kubernetes/assertions/TestAssertions.java | 26 ------------------- .../kubernetes/utils/CommonTestUtils.java | 22 ---------------- 3 files changed, 52 deletions(-) diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java index 610e2b2e6be..f889a893674 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java @@ -358,7 +358,6 @@ public void testErrorPathDomainMismatchMountPath() { // create domain and verify it is failed logger.info("Creating domain {0} with auxiliary image {1} in namespace {2}", domainUid, errorPathAuxiliaryImage1, errorpathDomainNamespace); - assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); // check the domain event contains the expected error msg @@ -421,7 +420,6 @@ public void testErrorPathDomainMissingWDTBinary() { // create domain and verify it is failed logger.info("Creating domain {0} with common mount image {1} in namespace {2}", domainUid, errorPathAuxiliaryImage2, errorpathDomainNamespace); - assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); // check the domain event contains the expected error msg @@ -488,7 +486,6 @@ public void testErrorPathDomainMissingDomainConfig() { // create domain and verify it is failed logger.info("Creating domain {0} with auxiliary image {1} in namespace {2}", domainUid, errorPathAuxiliaryImage3, errorpathDomainNamespace); - assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); // check the domain event contains the expected error msg @@ -564,7 +561,6 @@ public void testErrorPathDomainWithFailCustomMountCommand() { // create domain and verify it is failed logger.info("Creating domain {0} with auxiliary image {1} in namespace {2}", domainUid, errorPathAuxiliaryImage4, errorpathDomainNamespace); - assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java index f26b4f3a0e3..d5386855dc8 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java @@ -395,32 +395,6 @@ public static Callable domainStatusReasonMatches(oracle.weblogic.domain }; } - /** - * Check the status message of the domain matches the given message. - * @param domain oracle.weblogic.domain.Domain object - * @param statusMessage the expected status message of the domain - * @return true if the status message matches, false otherwise - */ - public static Callable domainStatusMessageMatches(oracle.weblogic.domain.Domain domain, - String statusMessage) { - LoggingFacade logger = getLogger(); - return () -> { - if (domain != null && domain.getStatus() != null && domain.getStatus().getReason() != null) { - logger.info("domain status reason: {0}", domain.getStatus().getReason()); - return domain.getStatus().getReason().equalsIgnoreCase(statusMessage); - } else { - if (domain == null) { - logger.info("domain is null"); - } else if (domain.getStatus() == null) { - logger.info("domain status is null"); - } else { - logger.info("domain status reason is null"); - } - return false; - } - }; - } - /** * Check if a loadbalancer pod is ready. * diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java index c9c3cc5214c..361f766b83e 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java @@ -223,7 +223,6 @@ import static oracle.weblogic.kubernetes.assertions.TestAssertions.doesImageExist; import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainDoesNotExist; import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainExists; -import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainStatusMessageMatches; import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainStatusReasonMatches; import static oracle.weblogic.kubernetes.assertions.TestAssertions.isApacheReady; import static oracle.weblogic.kubernetes.assertions.TestAssertions.isElkStackPodReady; @@ -2038,27 +2037,6 @@ public static void checkDomainStatusReasonMatches(Domain domain, String namespac .until(assertDoesNotThrow(() -> domainStatusReasonMatches(domain, statusReason))); } - /** - * Check the status message of the domain matches the given message. - * - * @param domain oracle.weblogic.domain.Domain object - * @param namespace the namespace in which the domain exists - * @param statusMessage the expected status message of the domain - */ - public static void checkDomainStatusMessageMatches(Domain domain, String namespace, String statusMessage) { - LoggingFacade logger = getLogger(); - withStandardRetryPolicy - .conditionEvaluationListener( - condition -> logger.info("Waiting for the status message of the domain {0} in namespace {1} " - + "is {2} (elapsed time {3}ms, remaining time {4}ms)", - domain, - namespace, - statusMessage, - condition.getElapsedTimeInMS(), - condition.getRemainingTimeInMS())) - .until(assertDoesNotThrow(() -> domainStatusMessageMatches(domain, statusMessage))); - } - /** * Check whether the cluster's replica count matches with input parameter value. * From d1c8bb030b509aa3a92648962fa6d644a7e7df9b Mon Sep 17 00:00:00 2001 From: xiancao Date: Wed, 14 Jul 2021 23:04:17 +0000 Subject: [PATCH 6/8] address Pani and Vanaja's review comments --- .../kubernetes/ItMiiAuxiliaryImage.java | 172 ++++++++++-------- .../kubernetes/ItParameterizedDomain.java | 28 +-- .../kubernetes/utils/CommonMiiTestUtils.java | 5 +- .../kubernetes/utils/CommonTestUtils.java | 131 +++++++++++++ 4 files changed, 231 insertions(+), 105 deletions(-) diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java index 91dea91fa75..ac49b905410 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java @@ -8,15 +8,12 @@ import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.time.OffsetDateTime; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import io.kubernetes.client.custom.V1Patch; -import io.kubernetes.client.openapi.models.CoreV1Event; import oracle.weblogic.domain.AuxiliaryImage; -import oracle.weblogic.domain.AuxiliaryImageVolume; import oracle.weblogic.domain.Domain; import oracle.weblogic.kubernetes.actions.impl.primitive.Command; import oracle.weblogic.kubernetes.actions.impl.primitive.CommandParams; @@ -35,6 +32,7 @@ import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; import static oracle.weblogic.kubernetes.TestConstants.ADMIN_PASSWORD_DEFAULT; +import static oracle.weblogic.kubernetes.TestConstants.ADMIN_SERVER_NAME_BASE; import static oracle.weblogic.kubernetes.TestConstants.ADMIN_USERNAME_DEFAULT; import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_IMAGES_REPO; import static oracle.weblogic.kubernetes.TestConstants.MII_AUXILIARY_IMAGE_NAME; @@ -42,6 +40,7 @@ import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_TAG; import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_WDT_MODEL_FILE; import static oracle.weblogic.kubernetes.TestConstants.OCIR_SECRET_NAME; +import static oracle.weblogic.kubernetes.TestConstants.OPERATOR_RELEASE_NAME; import static oracle.weblogic.kubernetes.TestConstants.RESULTS_ROOT; import static oracle.weblogic.kubernetes.TestConstants.WEBLOGIC_IMAGE_NAME; import static oracle.weblogic.kubernetes.TestConstants.WEBLOGIC_IMAGE_TAG; @@ -54,13 +53,17 @@ import static oracle.weblogic.kubernetes.actions.TestActions.deleteImage; import static oracle.weblogic.kubernetes.actions.TestActions.dockerPush; import static oracle.weblogic.kubernetes.actions.TestActions.getDomainCustomResource; +import static oracle.weblogic.kubernetes.actions.TestActions.getOperatorPodName; import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort; import static oracle.weblogic.kubernetes.actions.TestActions.now; import static oracle.weblogic.kubernetes.actions.TestActions.patchDomainCustomResource; import static oracle.weblogic.kubernetes.assertions.TestAssertions.secretExists; import static oracle.weblogic.kubernetes.assertions.TestAssertions.verifyRollingRestartOccurred; import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createDomainResource; -import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkEvent; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkDomainEventContainsExpectedMsg; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkIntrospectorPodLogContainsExpectedMsg; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodLogContainsString; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReady; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkSystemResourceConfig; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkSystemResourceConfiguration; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDomainAndVerify; @@ -74,7 +77,6 @@ import static oracle.weblogic.kubernetes.utils.FileUtils.replaceStringInFile; import static oracle.weblogic.kubernetes.utils.FileUtils.unzipWDTInstallationFile; import static oracle.weblogic.kubernetes.utils.K8sEvents.DOMAIN_PROCESSING_FAILED; -import static oracle.weblogic.kubernetes.utils.K8sEvents.getEvent; import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; import static org.awaitility.Awaitility.with; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @@ -328,6 +330,7 @@ public void testUpdateDataSourceInDomainUsingAuxiliaryImage() { * Negative Test to create domain with mismatch mount path in auxiliary image and auxiliaryImageVolumes. * in auxiliaryImageVolumes, set mountPath to "/errorpath" * in auxiliary image, set AUXILIARY_IMAGE_PATH to "/auxiliary" + * Check the error msg is in introspector log, domain events and operator log. */ @Test @Order(3) @@ -382,10 +385,18 @@ public void testErrorPathDomainMismatchMountPath() { domainUid, errorPathAuxiliaryImage1, errorpathDomainNamespace); assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); + // check the introspector log contains the expected error msg + String expectedErrorMsg = "Auxiliary Image: Dir '/errorpath' doesn't exist or is empty. Exiting."; + checkIntrospectorPodLogContainsExpectedMsg(domainUid, errorpathDomainNamespace, expectedErrorMsg); + // check the domain event contains the expected error msg - String expectedErrorMsg = "Failed to complete processing domain resource domain1 due to: " - + "Auxiliary Image: Dir '/errorpath' doesn't exist or is empty. Exiting."; - verifyDomainEventContainsExpectedErrorMsg(timestamp, expectedErrorMsg); + checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, + "Warning", timestamp, expectedErrorMsg); + + // check the operator log contains the expected error msg + String operatorPodName = + assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); + checkPodLogContainsString(opNamespace, operatorPodName, expectedErrorMsg); // delete domain1 deleteDomainResource(errorpathDomainNamespace, domainUid); @@ -393,6 +404,7 @@ public void testErrorPathDomainMismatchMountPath() { /** * Negative Test to create domain without WDT binary. + * Check the error msg in introspector log, domain events and operator log. */ @Test @Order(4) @@ -444,11 +456,20 @@ public void testErrorPathDomainMissingWDTBinary() { domainUid, errorPathAuxiliaryImage2, errorpathDomainNamespace); assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); - // check the domain event contains the expected error msg + // check the introspector log contains the expected error msg String expectedErrorMsg = "The domain resource 'spec.domainHomeSourceType' is 'FromModel' and " + "a WebLogic Deploy Tool (WDT) install is not located at 'spec.configuration.model.wdtInstallHome' " + "which is currently set to '/auxiliary/weblogic-deploy'"; - verifyDomainEventContainsExpectedErrorMsg(timestamp, expectedErrorMsg); + checkIntrospectorPodLogContainsExpectedMsg(domainUid, errorpathDomainNamespace, expectedErrorMsg); + + // check the domain event contains the expected error msg + checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, + "Warning", timestamp, expectedErrorMsg); + + // check the operator log contains the expected error msg + String operatorPodName = + assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); + checkPodLogContainsString(opNamespace, operatorPodName, expectedErrorMsg); // delete domain1 deleteDomainResource(errorpathDomainNamespace, domainUid); @@ -456,6 +477,7 @@ public void testErrorPathDomainMissingWDTBinary() { /** * Negative Test to create domain without domain model file, the auxiliary image contains only sparse JMS config. + * Check the error message in introspector log, domain events and operator log */ @Test @Order(5) @@ -513,14 +535,27 @@ public void testErrorPathDomainMissingDomainConfig() { // check the domain event contains the expected error msg String expectedErrorMsg = "createDomain did not find the required domainInfo section in the model file /auxiliary/models/model.jms2.yaml"; - verifyDomainEventContainsExpectedErrorMsg(timestamp, expectedErrorMsg); + checkIntrospectorPodLogContainsExpectedMsg(domainUid, errorpathDomainNamespace, expectedErrorMsg); + + // check the domain event contains the expected error msg + checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, + "Warning", timestamp, expectedErrorMsg); + + // check the operator log contains the expected error msg + String operatorPodName = + assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); + checkPodLogContainsString(opNamespace, operatorPodName, expectedErrorMsg); // delete domain1 deleteDomainResource(errorpathDomainNamespace, domainUid); } /** - * Negative Test to create domain using a custom mount command that's guaranteed to fail, for example: exit 1. + * Negative Test to patch the existing domain using a custom mount command that's guaranteed to fail. + * Specify domain.spec.serverPod.auxiliaryImages.command to a custom mount command instead of the default one, which + * defaults to "cp -R $AUXILIARY_IMAGE_PATH/* $TARGET_MOUNT_PATH" + * Check the error msg in admin server pod log, domain events and operator pod log. + * Restore the domain by removing the custom mount command. */ @Test @Order(6) @@ -528,71 +563,64 @@ public void testErrorPathDomainMissingDomainConfig() { public void testErrorPathDomainWithFailCustomMountCommand() { OffsetDateTime timestamp = now(); - String errorPathAuxiliaryImage4 = MII_AUXILIARY_IMAGE_NAME + ":errorpathimage4"; - - final String auxiliaryImageVolumeName = "auxiliaryImageVolume1"; - final String auxiliaryImagePath = "/auxiliary"; - createSecretsForDomain(adminSecretName, encryptionSecretName, errorpathDomainNamespace); - - // create stage dir for auxiliary image - Path errorpathAIPath4 = Paths.get(RESULTS_ROOT, "errorpathauxiimage4"); - assertDoesNotThrow(() -> FileUtils.deleteDirectory(errorpathAIPath4.toFile())); - assertDoesNotThrow(() -> Files.createDirectories(errorpathAIPath4)); - - // create models dir and copy model for image - Path modelsPath4 = Paths.get(errorpathAIPath4.toString(), "models"); - assertDoesNotThrow(() -> Files.createDirectories(modelsPath4)); - assertDoesNotThrow(() -> Files.copy( - Paths.get(MODEL_DIR, MII_BASIC_WDT_MODEL_FILE), - Paths.get(modelsPath4.toString(), MII_BASIC_WDT_MODEL_FILE), - StandardCopyOption.REPLACE_EXISTING)); - - // unzip WDT installation file into work dir - unzipWDTInstallationFile(errorpathAIPath4.toString()); + Domain domain1 = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, domainNamespace), + String.format("getDomainCustomResource failed with ApiException when tried to get domain %s in namespace %s", + domainUid, domainNamespace)); + assertNotNull(domain1, "Got null domain resource "); + assertNotNull(domain1.getSpec().getServerPod().getAuxiliaryImages(), + domain1 + "/spec/serverPod/auxiliaryImages is null"); - // create image1 with model and wdt installation files - createAuxiliaryImage(errorpathAIPath4.toString(), - Paths.get(RESOURCE_DIR, "auxiliaryimage", "Dockerfile").toString(), errorPathAuxiliaryImage4); + List auxiliaryImageList = domain1.getSpec().getServerPod().getAuxiliaryImages(); + assertFalse(auxiliaryImageList.isEmpty(), "AuxiliaryImage list is empty"); - // push image to repo for multi node cluster - if (!DOMAIN_IMAGES_REPO.isEmpty()) { - logger.info("docker push image {0} to registry {1}", errorPathAuxiliaryImage4, DOMAIN_IMAGES_REPO); - assertTrue(dockerPush(errorPathAuxiliaryImage4), - String.format("docker push failed for image %s", errorPathAuxiliaryImage4)); - } + // patch the first auxiliary image + String searchString = "\"/spec/serverPod/auxiliaryImages/0/command\""; + StringBuffer patchStr = new StringBuffer("[{"); + patchStr.append("\"op\": \"add\",") + .append(" \"path\": " + searchString + ",") + .append(" \"value\": \"exit 1\"") + .append(" }]"); + logger.info("Auxiliary Image patch string: " + patchStr); - // create domain custom resource using common mount and images - logger.info("Creating domain custom resource with domainUid {0} and auxiliary image {1}", - domainUid, errorPathAuxiliaryImage4); - AuxiliaryImageVolume auxiliaryImageVolume = new AuxiliaryImageVolume() - .name(auxiliaryImageVolumeName) - .mountPath(auxiliaryImagePath); + V1Patch patch = new V1Patch((patchStr).toString()); - AuxiliaryImage auxiliaryImage = new AuxiliaryImage() - .image(errorPathAuxiliaryImage4) - .imagePullPolicy("IfNotPresent") - .volume(auxiliaryImageVolumeName) - .command("exit 1"); + boolean aiPatched = assertDoesNotThrow(() -> + patchDomainCustomResource(domainUid, domainNamespace, patch, "application/json-patch+json"), + "patchDomainCustomResource(Auxiliary Image) failed "); + assertTrue(aiPatched, "patchDomainCustomResource(auxiliary image) failed"); - Domain domainCR = createDomainResource(domainUid, errorpathDomainNamespace, - WEBLOGIC_IMAGE_NAME + ":" + WEBLOGIC_IMAGE_TAG, adminSecretName, OCIR_SECRET_NAME, - encryptionSecretName, replicaCount, "cluster-1", Arrays.asList(auxiliaryImageVolume), - Arrays.asList(auxiliaryImage)); + // check the domain event contains the expected error msg + String expectedErrorMsg = "Auxiliary Image: Command 'exit 1' execution failed in container"; + checkDomainEventContainsExpectedMsg(opNamespace, domainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, + "Warning", timestamp, expectedErrorMsg); + + // check the admin server log contains the expected error msg + final String adminServerPodName = domainUid + "-" + ADMIN_SERVER_NAME_BASE; + checkPodLogContainsString(domainNamespace, adminServerPodName, expectedErrorMsg); + + // check the operator log contains the expected error msg + String operatorPodName = + assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); + checkPodLogContainsString(opNamespace, operatorPodName, expectedErrorMsg); + + // restore domain1 + // patch the first auxiliary image to remove the domain.spec.serverPod.auxiliaryImages.command + patchStr = new StringBuffer("[{"); + patchStr.append("\"op\": \"remove\",") + .append(" \"path\": " + searchString) + .append(" }]"); + logger.info("Auxiliary Image patch string: " + patchStr); - // create domain and verify it is failed - logger.info("Creating domain {0} with auxiliary image {1} in namespace {2}", - domainUid, errorPathAuxiliaryImage4, errorpathDomainNamespace); - assertDoesNotThrow(() -> createDomainCustomResource(domainCR), - "createDomainCustomResource throws Exception"); + V1Patch patch1 = new V1Patch((patchStr).toString()); - // check the domain event contains the expected error msg - String expectedErrorMsg = "Failed to complete processing domain resource domain1 due to: " - + "Auxiliary Image: Command 'exit 1' execution failed in container"; - verifyDomainEventContainsExpectedErrorMsg(timestamp, expectedErrorMsg); + aiPatched = assertDoesNotThrow(() -> + patchDomainCustomResource(domainUid, domainNamespace, patch1, "application/json-patch+json"), + "patchDomainCustomResource(Auxiliary Image) failed "); + assertTrue(aiPatched, "patchDomainCustomResource(auxiliary image) failed"); - // delete domain1 - deleteDomainResource(errorpathDomainNamespace, domainUid); + // check the admin server is up and running + checkPodReady(adminServerPodName, domainUid, domainNamespace); } private static void patchDomainWithAuxiliaryImageAndVerify(String oldImageName, String newImageName, @@ -707,12 +735,4 @@ private void createSecretsForDomain(String adminSecretName, String encryptionSec createSecretWithUsernamePassword(encryptionSecretName, domainNamespace, "weblogicenc", "weblogicenc"); } } - - private void verifyDomainEventContainsExpectedErrorMsg(OffsetDateTime timestamp, String expectedErrorMsg) { - checkEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); - CoreV1Event event = - getEvent(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp); - assertTrue(event.getMessage().contains(expectedErrorMsg), - String.format("The event message does not contain the expected error msg %s", expectedErrorMsg)); - } } diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItParameterizedDomain.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItParameterizedDomain.java index 2338893143d..d28e51c5ddc 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItParameterizedDomain.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItParameterizedDomain.java @@ -119,6 +119,7 @@ import static oracle.weblogic.kubernetes.assertions.TestAssertions.clusterRoleBindingExists; import static oracle.weblogic.kubernetes.assertions.TestAssertions.clusterRoleExists; import static oracle.weblogic.kubernetes.utils.CommonPatchTestUtils.patchDomainResource; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodLogContainsString; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReady; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDomainAndVerify; @@ -305,7 +306,7 @@ public void testOperatorLogSevereMsg() { createMiiDomainNegative("miidomainnegative", miiDomainNegativeNamespace); String operatorPodName = assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); - waitForPodLogContainsString(opNamespace, operatorPodName, + checkPodLogContainsString(opNamespace, operatorPodName, "Domain miidomainnegative is not valid: RuntimeEncryption secret '" + encryptionSecretName + "' not found in namespace '" + miiDomainNegativeNamespace + "'"); } @@ -1543,31 +1544,6 @@ private void waitForFileExistsInPod(String namespace, String podName, String fil "fileExistsInPod failed with IOException, ApiException or InterruptedException")); } - private Callable podLogContainsString(String namespace, String podName, String expectedString) { - return () -> { - logger.info("pod name: {0}", podName); - String podLog = getPodLog(podName, namespace); - logger.info("pod log: {0}", podLog); - return podLog.contains(expectedString); - }; - } - - private void waitForPodLogContainsString(String namespace, String podName, String expectedString) { - - logger.info("Wait for string {0} existing in pod {1} in namespace {2}", expectedString, podName, namespace); - withStandardRetryPolicy - .conditionEvaluationListener( - condition -> logger.info("Waiting for string {0} existing in pod {1} in namespace {2} " - + "(elapsed time {3}ms, remaining time {4}ms)", - expectedString, - podName, - namespace, - condition.getElapsedTimeInMS(), - condition.getRemainingTimeInMS())) - .until(assertDoesNotThrow(() -> podLogContainsString(namespace, podName, expectedString), - "podLogContainsString failed with IOException, ApiException or InterruptedException")); - } - /** * Negative test case for creating a model-in-image domain without encryption secret created. * The admin server service/pod will not be created. diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java index 4e6d29172c9..f10b280def8 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java @@ -300,8 +300,7 @@ public static Domain createDomainResource( } /** - * Create a domain object for a Kubernetes domain custom resource using the basic WLS image and MII auxiliary image - * image. + * Create a domain object for a Kubernetes domain custom resource using the basic WLS image and auxiliary image. * * @param domainResourceName name of the domain resource * @param domNamespace Kubernetes namespace that the domain is hosted @@ -315,7 +314,7 @@ public static Domain createDomainResource( * @param auxiliaryImages list of AuxiliaryImages * @return domain object of the domain resource */ - public static Domain createDomainResource( + public static Domain createDomainResourceWithAuxiliaryImage( String domainResourceName, String domNamespace, String baseImageName, diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java index 361f766b83e..50fb06af3c5 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java @@ -28,6 +28,7 @@ import io.kubernetes.client.custom.Quantity; import io.kubernetes.client.custom.V1Patch; import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.models.CoreV1Event; import io.kubernetes.client.openapi.models.NetworkingV1beta1HTTPIngressPath; import io.kubernetes.client.openapi.models.NetworkingV1beta1HTTPIngressRuleValue; import io.kubernetes.client.openapi.models.NetworkingV1beta1IngressBackend; @@ -192,6 +193,7 @@ import static oracle.weblogic.kubernetes.actions.TestActions.getOperatorImageName; import static oracle.weblogic.kubernetes.actions.TestActions.getPersistentVolume; import static oracle.weblogic.kubernetes.actions.TestActions.getPersistentVolumeClaim; +import static oracle.weblogic.kubernetes.actions.TestActions.getPod; import static oracle.weblogic.kubernetes.actions.TestActions.getPodCreationTimestamp; import static oracle.weblogic.kubernetes.actions.TestActions.getPodLog; import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort; @@ -250,6 +252,7 @@ import static oracle.weblogic.kubernetes.utils.ExecCommand.exec; import static oracle.weblogic.kubernetes.utils.FileUtils.checkDirectory; import static oracle.weblogic.kubernetes.utils.K8sEvents.checkDomainEvent; +import static oracle.weblogic.kubernetes.utils.K8sEvents.getEvent; import static oracle.weblogic.kubernetes.utils.TestUtils.callWebAppAndCheckForServerNameInResponse; import static oracle.weblogic.kubernetes.utils.TestUtils.callWebAppAndWaitTillReady; import static oracle.weblogic.kubernetes.utils.TestUtils.getNextFreePort; @@ -4376,4 +4379,132 @@ public static void deleteDomainResource(String domainNS, String domainUid) { condition.getRemainingTimeInMS())) .until(domainDoesNotExist(domainUid, DOMAIN_VERSION, domainNS)); } + + private static Callable podLogContainsString(String namespace, String podName, String expectedString) { + return () -> { + String podLog; + try { + podLog = getPodLog(podName, namespace); + getLogger().info("pod log for pod {0} in namespace {1} : {2}", podName, namespace, podLog); + } catch (ApiException apiEx) { + getLogger().severe("got ApiException while getting pod log: ", apiEx); + return false; + } + + return podLog.contains(expectedString); + }; + } + + /** + * Wait and check the pod log contains the expected string. + * @param namespace the namespace in which the pod exists + * @param podName the pod to get the log + * @param expectedString the expected string to check in the pod log + */ + public static void checkPodLogContainsString(String namespace, String podName, String expectedString) { + + getLogger().info("Wait for string {0} existing in pod {1} in namespace {2}", expectedString, podName, namespace); + withStandardRetryPolicy + .conditionEvaluationListener( + condition -> getLogger().info("Waiting for string {0} existing in pod {1} in namespace {2} " + + "(elapsed time {3}ms, remaining time {4}ms)", + expectedString, + podName, + namespace, + condition.getElapsedTimeInMS(), + condition.getRemainingTimeInMS())) + .until(assertDoesNotThrow(() -> podLogContainsString(namespace, podName, expectedString), + "podLogContainsString failed with IOException, ApiException or InterruptedException")); + } + + /** + * Check the domain event contains the expected error msg. + * + * @param opNamespace namespace in which the operator is running + * @param domainNamespace namespace in which the domain exists + * @param domainUid UID of the domain + * @param reason event to check for Created, Changed, deleted, processing etc + * @param type type of event, Normal of Warning + * @param timestamp the timestamp after which to see events + * @param expectedMsg the expected message in the domain event message + */ + public static void checkDomainEventContainsExpectedMsg(String opNamespace, + String domainNamespace, + String domainUid, + String reason, + String type, + OffsetDateTime timestamp, + String expectedMsg) { + checkEvent(opNamespace, domainNamespace, domainUid, reason, type, timestamp); + CoreV1Event event = + getEvent(opNamespace, domainNamespace, domainUid, reason, type, timestamp); + if (event != null && event.getMessage() != null) { + assertTrue(event.getMessage().contains(expectedMsg), + String.format("The event message does not contain the expected msg %s", expectedMsg)); + } else { + fail("event is null or event message is null"); + } + } + + /** + * Check the introspector pod log contains the expected message. + * @param domainUid domain uid of the domain + * @param domainNamespace namespace of the domain + * @param expectedMsg expected message in the introspector pod log + */ + public static void checkIntrospectorPodLogContainsExpectedMsg(String domainUid, + String domainNamespace, + String expectedMsg) { + // verify the introspector pod is created + String introspectJobName = getIntrospectJobName(domainUid); + + // check whether the introspector log contains the expected error message + getLogger().info("verifying that the introspector log contains the expected error message"); + withStandardRetryPolicy + .conditionEvaluationListener( + condition -> + getLogger().info( + "Checking for the log of introspector pod contains the expected msg {0}. " + + "Elapsed time {1}ms, remaining time {2}ms", + expectedMsg, + condition.getElapsedTimeInMS(), + condition.getRemainingTimeInMS())) + .until(() -> + introspectorPodLogContainsExpectedErrorMsg(introspectJobName, domainUid, domainNamespace, expectedMsg)); + } + + private static boolean introspectorPodLogContainsExpectedErrorMsg(String introspectJobName, + String domainUid, + String namespace, + String errormsg) { + String introspectPodName; + V1Pod introspectorPod; + + String labelSelector = String.format("weblogic.domainUID in (%s)", domainUid); + + try { + introspectorPod = getPod(namespace, labelSelector, introspectJobName); + } catch (ApiException apiEx) { + getLogger().severe("got ApiException while getting pod:", apiEx); + return false; + } + + if (introspectorPod != null && introspectorPod.getMetadata() != null) { + introspectPodName = introspectorPod.getMetadata().getName(); + } else { + return false; + } + + String introspectorLog; + try { + introspectorLog = getPodLog(introspectPodName, namespace); + getLogger().info("introspector log: {0}", introspectorLog); + } catch (ApiException apiEx) { + getLogger().severe("got ApiException while getting pod log:", apiEx); + return false; + } + + return introspectorLog.contains(errormsg); + } + } From 11c2c8077746d68773f26a4a1d4805a88ba5f1bc Mon Sep 17 00:00:00 2001 From: xiancao Date: Thu, 15 Jul 2021 02:44:13 +0000 Subject: [PATCH 7/8] clean up --- .../kubernetes/ItMiiAuxiliaryImage.java | 46 +++++----- .../kubernetes/utils/CommonTestUtils.java | 83 +++++-------------- 2 files changed, 45 insertions(+), 84 deletions(-) diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java index ac49b905410..b27f55ac5ef 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java @@ -32,7 +32,6 @@ import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; import static oracle.weblogic.kubernetes.TestConstants.ADMIN_PASSWORD_DEFAULT; -import static oracle.weblogic.kubernetes.TestConstants.ADMIN_SERVER_NAME_BASE; import static oracle.weblogic.kubernetes.TestConstants.ADMIN_USERNAME_DEFAULT; import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_IMAGES_REPO; import static oracle.weblogic.kubernetes.TestConstants.MII_AUXILIARY_IMAGE_NAME; @@ -61,7 +60,6 @@ import static oracle.weblogic.kubernetes.assertions.TestAssertions.verifyRollingRestartOccurred; import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createDomainResource; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkDomainEventContainsExpectedMsg; -import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkIntrospectorPodLogContainsExpectedMsg; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodLogContainsString; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReady; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkSystemResourceConfig; @@ -71,6 +69,7 @@ import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createSecretWithUsernamePassword; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.deleteDomainResource; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getExternalServicePodName; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getIntrospectorPodName; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getPodsWithTimeStamps; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyOperator; import static oracle.weblogic.kubernetes.utils.FileUtils.copyFileFromPod; @@ -330,7 +329,7 @@ public void testUpdateDataSourceInDomainUsingAuxiliaryImage() { * Negative Test to create domain with mismatch mount path in auxiliary image and auxiliaryImageVolumes. * in auxiliaryImageVolumes, set mountPath to "/errorpath" * in auxiliary image, set AUXILIARY_IMAGE_PATH to "/auxiliary" - * Check the error msg is in introspector log, domain events and operator log. + * Check the error msg is in introspector pod log, domain events and operator pod log. */ @Test @Order(3) @@ -385,15 +384,16 @@ public void testErrorPathDomainMismatchMountPath() { domainUid, errorPathAuxiliaryImage1, errorpathDomainNamespace); assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); - // check the introspector log contains the expected error msg + // check the introspector pod log contains the expected error msg String expectedErrorMsg = "Auxiliary Image: Dir '/errorpath' doesn't exist or is empty. Exiting."; - checkIntrospectorPodLogContainsExpectedMsg(domainUid, errorpathDomainNamespace, expectedErrorMsg); + String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace)); + checkPodLogContainsString(errorpathDomainNamespace, introspectorPodName, expectedErrorMsg); // check the domain event contains the expected error msg checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp, expectedErrorMsg); - // check the operator log contains the expected error msg + // check the operator pod log contains the expected error msg String operatorPodName = assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); checkPodLogContainsString(opNamespace, operatorPodName, expectedErrorMsg); @@ -404,7 +404,7 @@ public void testErrorPathDomainMismatchMountPath() { /** * Negative Test to create domain without WDT binary. - * Check the error msg in introspector log, domain events and operator log. + * Check the error msg is in introspector pod log, domain events and operator pod log. */ @Test @Order(4) @@ -456,17 +456,18 @@ public void testErrorPathDomainMissingWDTBinary() { domainUid, errorPathAuxiliaryImage2, errorpathDomainNamespace); assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); - // check the introspector log contains the expected error msg + // check the introspector pod log contains the expected error msg String expectedErrorMsg = "The domain resource 'spec.domainHomeSourceType' is 'FromModel' and " + "a WebLogic Deploy Tool (WDT) install is not located at 'spec.configuration.model.wdtInstallHome' " + "which is currently set to '/auxiliary/weblogic-deploy'"; - checkIntrospectorPodLogContainsExpectedMsg(domainUid, errorpathDomainNamespace, expectedErrorMsg); + String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace)); + checkPodLogContainsString(errorpathDomainNamespace, introspectorPodName, expectedErrorMsg); // check the domain event contains the expected error msg checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp, expectedErrorMsg); - // check the operator log contains the expected error msg + // check the operator pod log contains the expected error msg String operatorPodName = assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); checkPodLogContainsString(opNamespace, operatorPodName, expectedErrorMsg); @@ -477,7 +478,7 @@ public void testErrorPathDomainMissingWDTBinary() { /** * Negative Test to create domain without domain model file, the auxiliary image contains only sparse JMS config. - * Check the error message in introspector log, domain events and operator log + * Check the error message is in introspector pod log, domain events and operator pod log */ @Test @Order(5) @@ -532,16 +533,17 @@ public void testErrorPathDomainMissingDomainConfig() { domainUid, errorPathAuxiliaryImage3, errorpathDomainNamespace); assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); - // check the domain event contains the expected error msg + // check the introspector pod log contains the expected error msg String expectedErrorMsg = "createDomain did not find the required domainInfo section in the model file /auxiliary/models/model.jms2.yaml"; - checkIntrospectorPodLogContainsExpectedMsg(domainUid, errorpathDomainNamespace, expectedErrorMsg); + String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace)); + checkPodLogContainsString(errorpathDomainNamespace, introspectorPodName, expectedErrorMsg); // check the domain event contains the expected error msg checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp, expectedErrorMsg); - // check the operator log contains the expected error msg + // check the operator pod log contains the expected error msg String operatorPodName = assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); checkPodLogContainsString(opNamespace, operatorPodName, expectedErrorMsg); @@ -554,12 +556,12 @@ public void testErrorPathDomainMissingDomainConfig() { * Negative Test to patch the existing domain using a custom mount command that's guaranteed to fail. * Specify domain.spec.serverPod.auxiliaryImages.command to a custom mount command instead of the default one, which * defaults to "cp -R $AUXILIARY_IMAGE_PATH/* $TARGET_MOUNT_PATH" - * Check the error msg in admin server pod log, domain events and operator pod log. + * Check the error msg in introspector pod log, domain events and operator pod log. * Restore the domain by removing the custom mount command. */ @Test @Order(6) - @DisplayName("Negative Test to create domain using a custom mount command that's guaranteed to fail") + @DisplayName("Negative Test to patch domain using a custom mount command that's guaranteed to fail") public void testErrorPathDomainWithFailCustomMountCommand() { OffsetDateTime timestamp = now(); @@ -590,16 +592,16 @@ public void testErrorPathDomainWithFailCustomMountCommand() { "patchDomainCustomResource(Auxiliary Image) failed "); assertTrue(aiPatched, "patchDomainCustomResource(auxiliary image) failed"); - // check the domain event contains the expected error msg + // check the introspector pod log contains the expected error msg String expectedErrorMsg = "Auxiliary Image: Command 'exit 1' execution failed in container"; + String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, domainNamespace)); + checkPodLogContainsString(domainNamespace, introspectorPodName, expectedErrorMsg); + + // check the domain event contains the expected error msg checkDomainEventContainsExpectedMsg(opNamespace, domainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, "Warning", timestamp, expectedErrorMsg); - // check the admin server log contains the expected error msg - final String adminServerPodName = domainUid + "-" + ADMIN_SERVER_NAME_BASE; - checkPodLogContainsString(domainNamespace, adminServerPodName, expectedErrorMsg); - - // check the operator log contains the expected error msg + // check the operator pod log contains the expected error msg String operatorPodName = assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); checkPodLogContainsString(opNamespace, operatorPodName, expectedErrorMsg); diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java index 50fb06af3c5..908a33979ce 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java @@ -3847,6 +3847,27 @@ public static String getIntrospectJobName(String domainUid) { return domainUid + TestConstants.DEFAULT_INTROSPECTOR_JOB_NAME_SUFFIX; } + /** + * Get the introspector pod name. + * @param domainUid domain uid of the domain + * @param domainNamespace domain namespace in which introspector runs + * @return the introspector pod name + * @throws ApiException if Kubernetes API calls fail + */ + public static String getIntrospectorPodName(String domainUid, String domainNamespace) throws ApiException { + checkPodExists(getIntrospectJobName(domainUid), domainUid, domainNamespace); + + String labelSelector = String.format("weblogic.domainUID in (%s)", domainUid); + + V1Pod introspectorPod = getPod(domainNamespace, labelSelector, getIntrospectJobName(domainUid)); + + if (introspectorPod != null && introspectorPod.getMetadata() != null) { + return introspectorPod.getMetadata().getName(); + } else { + return ""; + } + } + /** * Set the inter-pod anti-affinity for the domain custom resource * so that server instances spread over the available Nodes. @@ -4445,66 +4466,4 @@ public static void checkDomainEventContainsExpectedMsg(String opNamespace, fail("event is null or event message is null"); } } - - /** - * Check the introspector pod log contains the expected message. - * @param domainUid domain uid of the domain - * @param domainNamespace namespace of the domain - * @param expectedMsg expected message in the introspector pod log - */ - public static void checkIntrospectorPodLogContainsExpectedMsg(String domainUid, - String domainNamespace, - String expectedMsg) { - // verify the introspector pod is created - String introspectJobName = getIntrospectJobName(domainUid); - - // check whether the introspector log contains the expected error message - getLogger().info("verifying that the introspector log contains the expected error message"); - withStandardRetryPolicy - .conditionEvaluationListener( - condition -> - getLogger().info( - "Checking for the log of introspector pod contains the expected msg {0}. " - + "Elapsed time {1}ms, remaining time {2}ms", - expectedMsg, - condition.getElapsedTimeInMS(), - condition.getRemainingTimeInMS())) - .until(() -> - introspectorPodLogContainsExpectedErrorMsg(introspectJobName, domainUid, domainNamespace, expectedMsg)); - } - - private static boolean introspectorPodLogContainsExpectedErrorMsg(String introspectJobName, - String domainUid, - String namespace, - String errormsg) { - String introspectPodName; - V1Pod introspectorPod; - - String labelSelector = String.format("weblogic.domainUID in (%s)", domainUid); - - try { - introspectorPod = getPod(namespace, labelSelector, introspectJobName); - } catch (ApiException apiEx) { - getLogger().severe("got ApiException while getting pod:", apiEx); - return false; - } - - if (introspectorPod != null && introspectorPod.getMetadata() != null) { - introspectPodName = introspectorPod.getMetadata().getName(); - } else { - return false; - } - - String introspectorLog; - try { - introspectorLog = getPodLog(introspectPodName, namespace); - getLogger().info("introspector log: {0}", introspectorLog); - } catch (ApiException apiEx) { - getLogger().severe("got ApiException while getting pod log:", apiEx); - return false; - } - - return introspectorLog.contains(errormsg); - } - } From af6172126ed581a348f5d24c156c91fb7b844420 Mon Sep 17 00:00:00 2001 From: xiancao Date: Thu, 15 Jul 2021 17:56:52 +0000 Subject: [PATCH 8/8] address Vanaja's comments --- .../kubernetes/ItMiiAuxiliaryImage.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java index b27f55ac5ef..ca531891eac 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java @@ -9,6 +9,7 @@ import java.nio.file.StandardCopyOption; import java.time.OffsetDateTime; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -60,8 +61,10 @@ import static oracle.weblogic.kubernetes.assertions.TestAssertions.verifyRollingRestartOccurred; import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createDomainResource; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkDomainEventContainsExpectedMsg; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodDoesNotExist; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodLogContainsString; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReady; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkServiceDoesNotExist; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkSystemResourceConfig; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkSystemResourceConfiguration; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDomainAndVerify; @@ -70,6 +73,7 @@ import static oracle.weblogic.kubernetes.utils.CommonTestUtils.deleteDomainResource; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getExternalServicePodName; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getIntrospectorPodName; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getPodCreationTime; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getPodsWithTimeStamps; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyOperator; import static oracle.weblogic.kubernetes.utils.FileUtils.copyFileFromPod; @@ -398,6 +402,14 @@ public void testErrorPathDomainMismatchMountPath() { assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); checkPodLogContainsString(opNamespace, operatorPodName, expectedErrorMsg); + // check there are no admin server and managed server pods and services created + checkPodDoesNotExist(adminServerPodName, domainUid, errorpathDomainNamespace); + checkServiceDoesNotExist(adminServerPodName, errorpathDomainNamespace); + for (int i = 1; i <= replicaCount; i++) { + checkPodDoesNotExist(managedServerPrefix + i, domainUid, errorpathDomainNamespace); + checkServiceDoesNotExist(managedServerPrefix + i, errorpathDomainNamespace); + } + // delete domain1 deleteDomainResource(errorpathDomainNamespace, domainUid); } @@ -472,6 +484,14 @@ public void testErrorPathDomainMissingWDTBinary() { assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); checkPodLogContainsString(opNamespace, operatorPodName, expectedErrorMsg); + // check there are no admin server and managed server pods and services created + checkPodDoesNotExist(adminServerPodName, domainUid, errorpathDomainNamespace); + checkServiceDoesNotExist(adminServerPodName, errorpathDomainNamespace); + for (int i = 1; i <= replicaCount; i++) { + checkPodDoesNotExist(managedServerPrefix + i, domainUid, errorpathDomainNamespace); + checkServiceDoesNotExist(managedServerPrefix + i, errorpathDomainNamespace); + } + // delete domain1 deleteDomainResource(errorpathDomainNamespace, domainUid); } @@ -548,6 +568,14 @@ public void testErrorPathDomainMissingDomainConfig() { assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); checkPodLogContainsString(opNamespace, operatorPodName, expectedErrorMsg); + // check there are no admin server and managed server pods and services created + checkPodDoesNotExist(adminServerPodName, domainUid, errorpathDomainNamespace); + checkServiceDoesNotExist(adminServerPodName, errorpathDomainNamespace); + for (int i = 1; i <= replicaCount; i++) { + checkPodDoesNotExist(managedServerPrefix + i, domainUid, errorpathDomainNamespace); + checkServiceDoesNotExist(managedServerPrefix + i, errorpathDomainNamespace); + } + // delete domain1 deleteDomainResource(errorpathDomainNamespace, domainUid); } @@ -566,6 +594,14 @@ public void testErrorPathDomainWithFailCustomMountCommand() { OffsetDateTime timestamp = now(); + // get the creation time of the admin server pod before patching + LinkedHashMap pods = new LinkedHashMap<>(); + pods.put(adminServerPodName, getPodCreationTime(domainNamespace, adminServerPodName)); + // get the creation time of the managed server pods before patching + for (int i = 1; i <= replicaCount; i++) { + pods.put(managedServerPrefix + i, getPodCreationTime(domainNamespace, managedServerPrefix + i)); + } + Domain domain1 = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, domainNamespace), String.format("getDomainCustomResource failed with ApiException when tried to get domain %s in namespace %s", domainUid, domainNamespace)); @@ -606,6 +642,18 @@ public void testErrorPathDomainWithFailCustomMountCommand() { assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); checkPodLogContainsString(opNamespace, operatorPodName, expectedErrorMsg); + // verify the domain is not rolled + // TODO: enable this check once https://jira.oraclecorp.com/jira/browse/OWLS-90971 is fixed + /* + logger.info("sleep 2 minutes to make sure the domain is not restarted"); + try { + Thread.sleep(120000); + } catch (InterruptedException ie) { + // ignore + } + verifyPodsNotRolled(domainNamespace, pods); + */ + // restore domain1 // patch the first auxiliary image to remove the domain.spec.serverPod.auxiliaryImages.command patchStr = new StringBuffer("[{");