diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItManageNameSpace.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItManageNameSpace.java index a00870531ba..613eb8fe243 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItManageNameSpace.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItManageNameSpace.java @@ -41,7 +41,6 @@ import static oracle.weblogic.kubernetes.TestConstants.ADMIN_SERVER_NAME_BASE; import static oracle.weblogic.kubernetes.TestConstants.ADMIN_USERNAME_DEFAULT; import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_API_VERSION; -import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_VERSION; import static oracle.weblogic.kubernetes.TestConstants.MANAGED_SERVER_NAME_BASE; import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_NAME; import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_TAG; @@ -58,12 +57,12 @@ import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort; import static oracle.weblogic.kubernetes.actions.TestActions.scaleClusterWithRestApi; import static oracle.weblogic.kubernetes.actions.TestActions.uninstallOperator; -import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainDoesNotExist; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodDoesNotExist; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDomainAndVerify; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createOcirRepoSecret; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createSecretWithUsernamePassword; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.deleteDomainResource; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyOperator; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.scaleAndVerifyCluster; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.setPodAntiAffinity; @@ -382,23 +381,6 @@ private void checkUpgradeFailedToAddNSManagedByAnotherOperator() { + " managed by other operator"); } - private void deleteDomainResource(String domainNS, String domainUid) { - //clean up domain resources in namespace and set namespace to label , managed by operator - logger.info("deleting domain custom resource {0}", domainUid); - assertTrue(deleteDomainCustomResource(domainUid, domainNS)); - - // wait until domain was deleted - withStandardRetryPolicy - .conditionEvaluationListener( - condition -> logger.info("Waiting for domain {0} to be deleted in namespace {1} " - + "(elapsed time {2}ms, remaining time {3}ms)", - domainUid, - domainNS, - condition.getElapsedTimeInMS(), - condition.getRemainingTimeInMS())) - .until(domainDoesNotExist(domainUid, DOMAIN_VERSION, domainNS)); - } - private HelmParams installAndVerifyOperatorCanManageDomainBySelector(Map managedDomains, Map unmanagedDomains, String selector, String selectorValue, diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java index 28bcbcf67a7..a66f5284e66 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java @@ -9,6 +9,7 @@ import java.nio.file.StandardCopyOption; import java.time.OffsetDateTime; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -41,6 +42,7 @@ import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_TAG; import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_WDT_MODEL_FILE; import static oracle.weblogic.kubernetes.TestConstants.OCIR_SECRET_NAME; +import static oracle.weblogic.kubernetes.TestConstants.OPERATOR_RELEASE_NAME; import static oracle.weblogic.kubernetes.TestConstants.RESULTS_ROOT; import static oracle.weblogic.kubernetes.TestConstants.WEBLOGIC_IMAGE_NAME; import static oracle.weblogic.kubernetes.TestConstants.WEBLOGIC_IMAGE_TAG; @@ -48,29 +50,42 @@ import static oracle.weblogic.kubernetes.actions.ActionConstants.MODEL_DIR; import static oracle.weblogic.kubernetes.actions.ActionConstants.RESOURCE_DIR; import static oracle.weblogic.kubernetes.actions.TestActions.buildAppArchive; +import static oracle.weblogic.kubernetes.actions.TestActions.createDomainCustomResource; import static oracle.weblogic.kubernetes.actions.TestActions.defaultAppParams; import static oracle.weblogic.kubernetes.actions.TestActions.deleteImage; import static oracle.weblogic.kubernetes.actions.TestActions.dockerPush; import static oracle.weblogic.kubernetes.actions.TestActions.dockerTag; import static oracle.weblogic.kubernetes.actions.TestActions.getDomainCustomResource; +import static oracle.weblogic.kubernetes.actions.TestActions.getOperatorPodName; import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort; +import static oracle.weblogic.kubernetes.actions.TestActions.now; import static oracle.weblogic.kubernetes.actions.TestActions.patchDomainCustomResource; +import static oracle.weblogic.kubernetes.assertions.TestAssertions.secretExists; import static oracle.weblogic.kubernetes.assertions.TestAssertions.verifyRollingRestartOccurred; import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createDomainResource; import static oracle.weblogic.kubernetes.utils.CommonPatchTestUtils.patchDomainResource; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkDomainEventContainsExpectedMsg; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodDoesNotExist; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodLogContainsString; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReady; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkServiceDoesNotExist; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkSystemResourceConfig; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkSystemResourceConfiguration; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDomainAndVerify; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createOcirRepoSecret; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createSecretWithUsernamePassword; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.deleteDomainResource; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.dockerLoginAndPushImageToRegistry; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getExternalServicePodName; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getIntrospectorPodName; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getPodCreationTime; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getPodsWithTimeStamps; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyOperator; import static oracle.weblogic.kubernetes.utils.FileUtils.copyFileFromPod; import static oracle.weblogic.kubernetes.utils.FileUtils.replaceStringInFile; import static oracle.weblogic.kubernetes.utils.FileUtils.unzipWDTInstallationFile; +import static oracle.weblogic.kubernetes.utils.K8sEvents.DOMAIN_PROCESSING_FAILED; import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; import static org.awaitility.Awaitility.with; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @@ -87,6 +102,7 @@ public class ItMiiAuxiliaryImage { private static String opNamespace = null; private static String domainNamespace = null; + private static String errorpathDomainNamespace = null; private static LoggingFacade logger = null; private String domainUid = "domain1"; private static String miiAuxiliaryImage1 = MII_AUXILIARY_IMAGE_NAME + ":" + MII_BASIC_IMAGE_TAG + "1"; @@ -96,6 +112,8 @@ public class ItMiiAuxiliaryImage { private final String adminServerPodName = domainUid + "-admin-server"; private final String managedServerPrefix = domainUid + "-managed-server"; private final int replicaCount = 2; + private String adminSecretName = "weblogic-credentials"; + private String encryptionSecretName = "encryptionsecret"; ConditionFactory withStandardRetryPolicy = with().pollDelay(0, SECONDS) @@ -108,7 +126,7 @@ public class ItMiiAuxiliaryImage { * JUnit engine parameter resolution mechanism */ @BeforeAll - public static void initAll(@Namespaces(2) List namespaces) { + public static void initAll(@Namespaces(3) List namespaces) { logger = getLogger(); // get a new unique opNamespace logger.info("Creating unique namespace for Operator"); @@ -119,8 +137,12 @@ public static void initAll(@Namespaces(2) List namespaces) { assertNotNull(namespaces.get(1), "Namespace list is null"); domainNamespace = namespaces.get(1); + logger.info("Creating unique namespace for errorpathDomain"); + assertNotNull(namespaces.get(2), "Namespace list is null"); + errorpathDomainNamespace = namespaces.get(2); + // install and verify operator - installAndVerifyOperator(opNamespace, domainNamespace); + installAndVerifyOperator(opNamespace, domainNamespace, errorpathDomainNamespace); } @@ -372,6 +394,350 @@ public void testUpdateBaseImageName() { } + /** + * Negative Test to create domain with mismatch mount path in auxiliary image and auxiliaryImageVolumes. + * in auxiliaryImageVolumes, set mountPath to "/errorpath" + * in auxiliary image, set AUXILIARY_IMAGE_PATH to "/auxiliary" + * Check the error msg is in introspector pod log, domain events and operator pod log. + */ + @Test + @Order(3) + @DisplayName("Negative Test to create domain with mismatch mount path in auxiliary image and auxiliaryImageVolumes") + public void testErrorPathDomainMismatchMountPath() { + + OffsetDateTime timestamp = now(); + String errorPathAuxiliaryImage1 = MII_AUXILIARY_IMAGE_NAME + ":errorpathimage1"; + + final String auxiliaryImageVolumeName = "auxiliaryImageVolume1"; + final String auxiliaryImagePath = "/errorpath"; + + createSecretsForDomain(adminSecretName, encryptionSecretName, errorpathDomainNamespace); + + // create stage dir for auxiliary image + Path errorpathAIPath1 = Paths.get(RESULTS_ROOT, "errorpathauxiimage1"); + assertDoesNotThrow(() -> FileUtils.deleteDirectory(errorpathAIPath1.toFile())); + assertDoesNotThrow(() -> Files.createDirectories(errorpathAIPath1)); + + // create models dir and copy model for image + Path modelsPath1 = Paths.get(errorpathAIPath1.toString(), "models"); + assertDoesNotThrow(() -> Files.createDirectories(modelsPath1)); + assertDoesNotThrow(() -> Files.copy( + Paths.get(MODEL_DIR, MII_BASIC_WDT_MODEL_FILE), + Paths.get(modelsPath1.toString(), MII_BASIC_WDT_MODEL_FILE), + StandardCopyOption.REPLACE_EXISTING)); + + // unzip WDT installation file into work dir + unzipWDTInstallationFile(errorpathAIPath1.toString()); + + // create image with model and wdt installation files + createAuxiliaryImage(errorpathAIPath1.toString(), + Paths.get(RESOURCE_DIR, "auxiliaryimage", "Dockerfile").toString(), errorPathAuxiliaryImage1); + + // push image to repo for multi node cluster + if (!DOMAIN_IMAGES_REPO.isEmpty()) { + logger.info("docker push image {0} to registry {1}", errorPathAuxiliaryImage1, DOMAIN_IMAGES_REPO); + assertTrue(dockerPush(errorPathAuxiliaryImage1), + String.format("docker push failed for image %s", errorPathAuxiliaryImage1)); + } + + // create domain custom resource using auxiliary images + logger.info("Creating domain custom resource with domainUid {0} and auxiliary image {1}", + domainUid, errorPathAuxiliaryImage1); + Domain domainCR = createDomainResource(domainUid, errorpathDomainNamespace, + WEBLOGIC_IMAGE_NAME + ":" + WEBLOGIC_IMAGE_TAG, adminSecretName, OCIR_SECRET_NAME, + encryptionSecretName, replicaCount, "cluster-1", auxiliaryImagePath, + auxiliaryImageVolumeName, errorPathAuxiliaryImage1); + + // create domain and verify it is failed + logger.info("Creating domain {0} with auxiliary image {1} in namespace {2}", + domainUid, errorPathAuxiliaryImage1, errorpathDomainNamespace); + assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); + + // check the introspector pod log contains the expected error msg + String expectedErrorMsg = "Auxiliary Image: Dir '/errorpath' doesn't exist or is empty. Exiting."; + String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace)); + checkPodLogContainsString(errorpathDomainNamespace, introspectorPodName, expectedErrorMsg); + + // check the domain event contains the expected error msg + checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, + "Warning", timestamp, expectedErrorMsg); + + // check the operator pod log contains the expected error msg + String operatorPodName = + assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); + checkPodLogContainsString(opNamespace, operatorPodName, expectedErrorMsg); + + // check there are no admin server and managed server pods and services created + checkPodDoesNotExist(adminServerPodName, domainUid, errorpathDomainNamespace); + checkServiceDoesNotExist(adminServerPodName, errorpathDomainNamespace); + for (int i = 1; i <= replicaCount; i++) { + checkPodDoesNotExist(managedServerPrefix + i, domainUid, errorpathDomainNamespace); + checkServiceDoesNotExist(managedServerPrefix + i, errorpathDomainNamespace); + } + + // delete domain1 + deleteDomainResource(errorpathDomainNamespace, domainUid); + } + + /** + * Negative Test to create domain without WDT binary. + * Check the error msg is in introspector pod log, domain events and operator pod log. + */ + @Test + @Order(4) + @DisplayName("Negative Test to create domain without WDT binary") + public void testErrorPathDomainMissingWDTBinary() { + + OffsetDateTime timestamp = now(); + String errorPathAuxiliaryImage2 = MII_AUXILIARY_IMAGE_NAME + ":errorpathimage2"; + + final String auxiliaryImageVolumeName = "auxiliaryImageVolume1"; + final String auxiliaryImagePath = "/auxiliary"; + + createSecretsForDomain(adminSecretName, encryptionSecretName, errorpathDomainNamespace); + + // create stage dir for common mount image + Path errorpathAIPath2 = Paths.get(RESULTS_ROOT, "errorpathauxiimage2"); + assertDoesNotThrow(() -> FileUtils.deleteDirectory(errorpathAIPath2.toFile())); + assertDoesNotThrow(() -> Files.createDirectories(errorpathAIPath2)); + + // create models dir and copy model for image + Path modelsPath2 = Paths.get(errorpathAIPath2.toString(), "models"); + assertDoesNotThrow(() -> Files.createDirectories(modelsPath2)); + assertDoesNotThrow(() -> Files.copy( + Paths.get(MODEL_DIR, MII_BASIC_WDT_MODEL_FILE), + Paths.get(modelsPath2.toString(), MII_BASIC_WDT_MODEL_FILE), + StandardCopyOption.REPLACE_EXISTING)); + + // create image with model and no wdt installation files + createAuxiliaryImage(errorpathAIPath2.toString(), + Paths.get(RESOURCE_DIR, "auxiliaryimage", "Dockerfile").toString(), errorPathAuxiliaryImage2); + + // push image to repo for multi node cluster + if (!DOMAIN_IMAGES_REPO.isEmpty()) { + logger.info("docker push image {0} to registry {1}", errorPathAuxiliaryImage2, DOMAIN_IMAGES_REPO); + assertTrue(dockerPush(errorPathAuxiliaryImage2), + String.format("docker push failed for image %s", errorPathAuxiliaryImage2)); + } + + // create domain custom resource using auxiliary images + logger.info("Creating domain custom resource with domainUid {0} and auxiliary image {1}", + domainUid, errorPathAuxiliaryImage2); + Domain domainCR = createDomainResource(domainUid, errorpathDomainNamespace, + WEBLOGIC_IMAGE_NAME + ":" + WEBLOGIC_IMAGE_TAG, adminSecretName, OCIR_SECRET_NAME, + encryptionSecretName, replicaCount, "cluster-1", auxiliaryImagePath, + auxiliaryImageVolumeName, errorPathAuxiliaryImage2); + + // create domain and verify it is failed + logger.info("Creating domain {0} with common mount image {1} in namespace {2}", + domainUid, errorPathAuxiliaryImage2, errorpathDomainNamespace); + assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); + + // check the introspector pod log contains the expected error msg + String expectedErrorMsg = "The domain resource 'spec.domainHomeSourceType' is 'FromModel' and " + + "a WebLogic Deploy Tool (WDT) install is not located at 'spec.configuration.model.wdtInstallHome' " + + "which is currently set to '/auxiliary/weblogic-deploy'"; + String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace)); + checkPodLogContainsString(errorpathDomainNamespace, introspectorPodName, expectedErrorMsg); + + // check the domain event contains the expected error msg + checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, + "Warning", timestamp, expectedErrorMsg); + + // check the operator pod log contains the expected error msg + String operatorPodName = + assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); + checkPodLogContainsString(opNamespace, operatorPodName, expectedErrorMsg); + + // check there are no admin server and managed server pods and services created + checkPodDoesNotExist(adminServerPodName, domainUid, errorpathDomainNamespace); + checkServiceDoesNotExist(adminServerPodName, errorpathDomainNamespace); + for (int i = 1; i <= replicaCount; i++) { + checkPodDoesNotExist(managedServerPrefix + i, domainUid, errorpathDomainNamespace); + checkServiceDoesNotExist(managedServerPrefix + i, errorpathDomainNamespace); + } + + // delete domain1 + deleteDomainResource(errorpathDomainNamespace, domainUid); + } + + /** + * Negative Test to create domain without domain model file, the auxiliary image contains only sparse JMS config. + * Check the error message is in introspector pod log, domain events and operator pod log + */ + @Test + @Order(5) + @DisplayName("Negative Test to create domain without domain model file, only having sparse JMS config") + public void testErrorPathDomainMissingDomainConfig() { + + OffsetDateTime timestamp = now(); + String errorPathAuxiliaryImage3 = MII_AUXILIARY_IMAGE_NAME + ":errorpathimage3"; + + final String auxiliaryImageVolumeName = "auxiliaryImageVolume1"; + final String auxiliaryImagePath = "/auxiliary"; + + createSecretsForDomain(adminSecretName, encryptionSecretName, errorpathDomainNamespace); + + // create stage dir for auxiliary image + Path errorpathAIPath3 = Paths.get(RESULTS_ROOT, "errorpathauxiimage3"); + assertDoesNotThrow(() -> FileUtils.deleteDirectory(errorpathAIPath3.toFile())); + assertDoesNotThrow(() -> Files.createDirectories(errorpathAIPath3)); + + // create models dir and copy model for image + Path modelsPath3 = Paths.get(errorpathAIPath3.toString(), "models"); + assertDoesNotThrow(() -> Files.createDirectories(modelsPath3)); + assertDoesNotThrow(() -> Files.copy( + Paths.get(MODEL_DIR, "model.jms2.yaml"), + Paths.get(modelsPath3.toString(), "model.jms2.yaml"), + StandardCopyOption.REPLACE_EXISTING)); + + // unzip WDT installation file into work dir + unzipWDTInstallationFile(errorpathAIPath3.toString()); + + // create image1 with model and wdt installation files + createAuxiliaryImage(errorpathAIPath3.toString(), + Paths.get(RESOURCE_DIR, "auxiliaryimage", "Dockerfile").toString(), errorPathAuxiliaryImage3); + + // push image to repo for multi node cluster + if (!DOMAIN_IMAGES_REPO.isEmpty()) { + logger.info("docker push image {0} to registry {1}", errorPathAuxiliaryImage3, DOMAIN_IMAGES_REPO); + assertTrue(dockerPush(errorPathAuxiliaryImage3), + String.format("docker push failed for image %s", errorPathAuxiliaryImage3)); + } + + // create domain custom resource using auxiliary images + logger.info("Creating domain custom resource with domainUid {0} and auxiliary image {1}", + domainUid, errorPathAuxiliaryImage3); + Domain domainCR = createDomainResource(domainUid, errorpathDomainNamespace, + WEBLOGIC_IMAGE_NAME + ":" + WEBLOGIC_IMAGE_TAG, adminSecretName, OCIR_SECRET_NAME, + encryptionSecretName, replicaCount, "cluster-1", auxiliaryImagePath, + auxiliaryImageVolumeName, errorPathAuxiliaryImage3); + + // create domain and verify it is failed + logger.info("Creating domain {0} with auxiliary image {1} in namespace {2}", + domainUid, errorPathAuxiliaryImage3, errorpathDomainNamespace); + assertDoesNotThrow(() -> createDomainCustomResource(domainCR), "createDomainCustomResource throws Exception"); + + // check the introspector pod log contains the expected error msg + String expectedErrorMsg = + "createDomain did not find the required domainInfo section in the model file /auxiliary/models/model.jms2.yaml"; + String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, errorpathDomainNamespace)); + checkPodLogContainsString(errorpathDomainNamespace, introspectorPodName, expectedErrorMsg); + + // check the domain event contains the expected error msg + checkDomainEventContainsExpectedMsg(opNamespace, errorpathDomainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, + "Warning", timestamp, expectedErrorMsg); + + // check the operator pod log contains the expected error msg + String operatorPodName = + assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); + checkPodLogContainsString(opNamespace, operatorPodName, expectedErrorMsg); + + // check there are no admin server and managed server pods and services created + checkPodDoesNotExist(adminServerPodName, domainUid, errorpathDomainNamespace); + checkServiceDoesNotExist(adminServerPodName, errorpathDomainNamespace); + for (int i = 1; i <= replicaCount; i++) { + checkPodDoesNotExist(managedServerPrefix + i, domainUid, errorpathDomainNamespace); + checkServiceDoesNotExist(managedServerPrefix + i, errorpathDomainNamespace); + } + + // delete domain1 + deleteDomainResource(errorpathDomainNamespace, domainUid); + } + + /** + * Negative Test to patch the existing domain using a custom mount command that's guaranteed to fail. + * Specify domain.spec.serverPod.auxiliaryImages.command to a custom mount command instead of the default one, which + * defaults to "cp -R $AUXILIARY_IMAGE_PATH/* $TARGET_MOUNT_PATH" + * Check the error msg in introspector pod log, domain events and operator pod log. + * Restore the domain by removing the custom mount command. + */ + @Test + @Order(6) + @DisplayName("Negative Test to patch domain using a custom mount command that's guaranteed to fail") + public void testErrorPathDomainWithFailCustomMountCommand() { + + OffsetDateTime timestamp = now(); + + // get the creation time of the admin server pod before patching + LinkedHashMap pods = new LinkedHashMap<>(); + pods.put(adminServerPodName, getPodCreationTime(domainNamespace, adminServerPodName)); + // get the creation time of the managed server pods before patching + for (int i = 1; i <= replicaCount; i++) { + pods.put(managedServerPrefix + i, getPodCreationTime(domainNamespace, managedServerPrefix + i)); + } + + Domain domain1 = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, domainNamespace), + String.format("getDomainCustomResource failed with ApiException when tried to get domain %s in namespace %s", + domainUid, domainNamespace)); + assertNotNull(domain1, "Got null domain resource "); + assertNotNull(domain1.getSpec().getServerPod().getAuxiliaryImages(), + domain1 + "/spec/serverPod/auxiliaryImages is null"); + + List auxiliaryImageList = domain1.getSpec().getServerPod().getAuxiliaryImages(); + assertFalse(auxiliaryImageList.isEmpty(), "AuxiliaryImage list is empty"); + + // patch the first auxiliary image + String searchString = "\"/spec/serverPod/auxiliaryImages/0/command\""; + StringBuffer patchStr = new StringBuffer("[{"); + patchStr.append("\"op\": \"add\",") + .append(" \"path\": " + searchString + ",") + .append(" \"value\": \"exit 1\"") + .append(" }]"); + logger.info("Auxiliary Image patch string: " + patchStr); + + V1Patch patch = new V1Patch((patchStr).toString()); + + boolean aiPatched = assertDoesNotThrow(() -> + patchDomainCustomResource(domainUid, domainNamespace, patch, "application/json-patch+json"), + "patchDomainCustomResource(Auxiliary Image) failed "); + assertTrue(aiPatched, "patchDomainCustomResource(auxiliary image) failed"); + + // check the introspector pod log contains the expected error msg + String expectedErrorMsg = "Auxiliary Image: Command 'exit 1' execution failed in container"; + String introspectorPodName = assertDoesNotThrow(() -> getIntrospectorPodName(domainUid, domainNamespace)); + checkPodLogContainsString(domainNamespace, introspectorPodName, expectedErrorMsg); + + // check the domain event contains the expected error msg + checkDomainEventContainsExpectedMsg(opNamespace, domainNamespace, domainUid, DOMAIN_PROCESSING_FAILED, + "Warning", timestamp, expectedErrorMsg); + + // check the operator pod log contains the expected error msg + String operatorPodName = + assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); + checkPodLogContainsString(opNamespace, operatorPodName, expectedErrorMsg); + + // verify the domain is not rolled + // TODO: enable this check once https://jira.oraclecorp.com/jira/browse/OWLS-90971 is fixed + /* + logger.info("sleep 2 minutes to make sure the domain is not restarted"); + try { + Thread.sleep(120000); + } catch (InterruptedException ie) { + // ignore + } + verifyPodsNotRolled(domainNamespace, pods); + */ + + // restore domain1 + // patch the first auxiliary image to remove the domain.spec.serverPod.auxiliaryImages.command + patchStr = new StringBuffer("[{"); + patchStr.append("\"op\": \"remove\",") + .append(" \"path\": " + searchString) + .append(" }]"); + logger.info("Auxiliary Image patch string: " + patchStr); + + V1Patch patch1 = new V1Patch((patchStr).toString()); + + aiPatched = assertDoesNotThrow(() -> + patchDomainCustomResource(domainUid, domainNamespace, patch1, "application/json-patch+json"), + "patchDomainCustomResource(Auxiliary Image) failed "); + assertTrue(aiPatched, "patchDomainCustomResource(auxiliary image) failed"); + + // check the admin server is up and running + checkPodReady(adminServerPodName, domainUid, domainNamespace); + } + private static void patchDomainWithAuxiliaryImageAndVerify(String oldImageName, String newImageName, String domainUid, String domainNamespace) { String adminServerPodName = domainUid + "-admin-server"; @@ -466,6 +832,25 @@ private void createAuxiliaryImage(String stageDirPath, String dockerFileLocation .execute(), String.format("Failed to execute", cmdToExecute)); } + private void createSecretsForDomain(String adminSecretName, String encryptionSecretName, String domainNamespace) { + if (!secretExists(OCIR_SECRET_NAME, domainNamespace)) { + createOcirRepoSecret(domainNamespace); + } + + // create secret for admin credentials + logger.info("Create secret for admin credentials"); + if (!secretExists(adminSecretName, domainNamespace)) { + createSecretWithUsernamePassword(adminSecretName, domainNamespace, + ADMIN_USERNAME_DEFAULT, ADMIN_PASSWORD_DEFAULT); + } + + // create encryption secret + logger.info("Create encryption secret"); + if (!secretExists(encryptionSecretName, domainNamespace)) { + createSecretWithUsernamePassword(encryptionSecretName, domainNamespace, "weblogicenc", "weblogicenc"); + } + } + private void checkConfiguredJMSresouce(String domainNamespace, String adminServerPodName) { int adminServiceNodePort = getServiceNodePort(domainNamespace, getExternalServicePodName(adminServerPodName), "default"); diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItParameterizedDomain.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItParameterizedDomain.java index 9d1a92ef40f..b3cc462c6e4 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItParameterizedDomain.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItParameterizedDomain.java @@ -119,6 +119,7 @@ import static oracle.weblogic.kubernetes.assertions.TestAssertions.clusterRoleBindingExists; import static oracle.weblogic.kubernetes.assertions.TestAssertions.clusterRoleExists; import static oracle.weblogic.kubernetes.utils.CommonPatchTestUtils.patchDomainResource; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodLogContainsString; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReady; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDomainAndVerify; @@ -306,7 +307,7 @@ public void testOperatorLogSevereMsg() { createMiiDomainNegative("miidomainnegative", miiDomainNegativeNamespace); String operatorPodName = assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); - waitForPodLogContainsString(opNamespace, operatorPodName, + checkPodLogContainsString(opNamespace, operatorPodName, "Domain miidomainnegative is not valid: RuntimeEncryption secret '" + encryptionSecretName + "' not found in namespace '" + miiDomainNegativeNamespace + "'"); } @@ -1545,31 +1546,6 @@ private void waitForFileExistsInPod(String namespace, String podName, String fil "fileExistsInPod failed with IOException, ApiException or InterruptedException")); } - private Callable podLogContainsString(String namespace, String podName, String expectedString) { - return () -> { - logger.info("pod name: {0}", podName); - String podLog = getPodLog(podName, namespace); - logger.info("pod log: {0}", podLog); - return podLog.contains(expectedString); - }; - } - - private void waitForPodLogContainsString(String namespace, String podName, String expectedString) { - - logger.info("Wait for string {0} existing in pod {1} in namespace {2}", expectedString, podName, namespace); - withStandardRetryPolicy - .conditionEvaluationListener( - condition -> logger.info("Waiting for string {0} existing in pod {1} in namespace {2} " - + "(elapsed time {3}ms, remaining time {4}ms)", - expectedString, - podName, - namespace, - condition.getElapsedTimeInMS(), - condition.getRemainingTimeInMS())) - .until(assertDoesNotThrow(() -> podLogContainsString(namespace, podName, expectedString), - "podLogContainsString failed with IOException, ApiException or InterruptedException")); - } - /** * Negative test case for creating a model-in-image domain without encryption secret created. * The admin server service/pod will not be created. diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java index 9e06b556a99..f10b280def8 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java @@ -299,6 +299,51 @@ public static Domain createDomainResource( return domainCR; } + /** + * Create a domain object for a Kubernetes domain custom resource using the basic WLS image and auxiliary image. + * + * @param domainResourceName name of the domain resource + * @param domNamespace Kubernetes namespace that the domain is hosted + * @param baseImageName name of the base image to use + * @param adminSecretName name of the new WebLogic admin credentials secret + * @param repoSecretName name of the secret for pulling the WebLogic image + * @param encryptionSecretName name of the secret used to encrypt the models + * @param replicaCount number of managed servers to start + * @param clusterName name of the cluster to add in domain + * @param auxiliaryImageVolumes list of AuxiliaryImageVolumes + * @param auxiliaryImages list of AuxiliaryImages + * @return domain object of the domain resource + */ + public static Domain createDomainResourceWithAuxiliaryImage( + String domainResourceName, + String domNamespace, + String baseImageName, + String adminSecretName, + String repoSecretName, + String encryptionSecretName, + int replicaCount, + String clusterName, + List auxiliaryImageVolumes, + List auxiliaryImages) { + + Domain domainCR = CommonMiiTestUtils.createDomainResource(domainResourceName, domNamespace, + baseImageName, adminSecretName, repoSecretName, + encryptionSecretName, replicaCount, clusterName); + + for (AuxiliaryImageVolume auxiliaryImageVolume : auxiliaryImageVolumes) { + domainCR.spec().addAuxiliaryImageVolumesItem(auxiliaryImageVolume); + domainCR.spec().configuration().model() + .withModelHome(auxiliaryImageVolume.getMountPath() + "/models") + .withWdtInstallHome(auxiliaryImageVolume.getMountPath() + "/weblogic-deploy"); + } + + for (AuxiliaryImage auxiliaryImage : auxiliaryImages) { + domainCR.spec().serverPod().addAuxiliaryImagesItem(auxiliaryImage); + } + + return domainCR; + } + /** * Create a domain object for a Kubernetes domain custom resource using the basic model-in-image * image. diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java index 871645a6bd2..194e6dae035 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java @@ -28,6 +28,7 @@ import io.kubernetes.client.custom.Quantity; import io.kubernetes.client.custom.V1Patch; import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.models.CoreV1Event; import io.kubernetes.client.openapi.models.NetworkingV1beta1HTTPIngressPath; import io.kubernetes.client.openapi.models.NetworkingV1beta1HTTPIngressRuleValue; import io.kubernetes.client.openapi.models.NetworkingV1beta1IngressBackend; @@ -185,12 +186,14 @@ import static oracle.weblogic.kubernetes.actions.TestActions.createService; import static oracle.weblogic.kubernetes.actions.TestActions.createServiceAccount; import static oracle.weblogic.kubernetes.actions.TestActions.defaultAppParams; +import static oracle.weblogic.kubernetes.actions.TestActions.deleteDomainCustomResource; import static oracle.weblogic.kubernetes.actions.TestActions.dockerLogin; import static oracle.weblogic.kubernetes.actions.TestActions.dockerPush; import static oracle.weblogic.kubernetes.actions.TestActions.getJob; import static oracle.weblogic.kubernetes.actions.TestActions.getOperatorImageName; import static oracle.weblogic.kubernetes.actions.TestActions.getPersistentVolume; import static oracle.weblogic.kubernetes.actions.TestActions.getPersistentVolumeClaim; +import static oracle.weblogic.kubernetes.actions.TestActions.getPod; import static oracle.weblogic.kubernetes.actions.TestActions.getPodCreationTimestamp; import static oracle.weblogic.kubernetes.actions.TestActions.getPodLog; import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort; @@ -220,6 +223,7 @@ import static oracle.weblogic.kubernetes.assertions.TestAssertions.credentialsNotValid; import static oracle.weblogic.kubernetes.assertions.TestAssertions.credentialsValid; import static oracle.weblogic.kubernetes.assertions.TestAssertions.doesImageExist; +import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainDoesNotExist; import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainExists; import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainStatusReasonMatches; import static oracle.weblogic.kubernetes.assertions.TestAssertions.isApacheReady; @@ -247,6 +251,8 @@ import static oracle.weblogic.kubernetes.assertions.TestAssertions.serviceExists; import static oracle.weblogic.kubernetes.utils.ExecCommand.exec; import static oracle.weblogic.kubernetes.utils.FileUtils.checkDirectory; +import static oracle.weblogic.kubernetes.utils.K8sEvents.checkDomainEvent; +import static oracle.weblogic.kubernetes.utils.K8sEvents.getEvent; import static oracle.weblogic.kubernetes.utils.TestUtils.callWebAppAndCheckForServerNameInResponse; import static oracle.weblogic.kubernetes.utils.TestUtils.callWebAppAndWaitTillReady; import static oracle.weblogic.kubernetes.utils.TestUtils.getNextFreePort; @@ -3866,6 +3872,27 @@ public static String getIntrospectJobName(String domainUid) { return domainUid + TestConstants.DEFAULT_INTROSPECTOR_JOB_NAME_SUFFIX; } + /** + * Get the introspector pod name. + * @param domainUid domain uid of the domain + * @param domainNamespace domain namespace in which introspector runs + * @return the introspector pod name + * @throws ApiException if Kubernetes API calls fail + */ + public static String getIntrospectorPodName(String domainUid, String domainNamespace) throws ApiException { + checkPodExists(getIntrospectJobName(domainUid), domainUid, domainNamespace); + + String labelSelector = String.format("weblogic.domainUID in (%s)", domainUid); + + V1Pod introspectorPod = getPod(domainNamespace, labelSelector, getIntrospectJobName(domainUid)); + + if (introspectorPod != null && introspectorPod.getMetadata() != null) { + return introspectorPod.getMetadata().getName(); + } else { + return ""; + } + } + /** * Set the inter-pod anti-affinity for the domain custom resource * so that server instances spread over the available Nodes. @@ -4352,4 +4379,116 @@ public static String getDockerExtraArgs() { } return extraArgs.toString(); } + + /** + * Wait until a given event is logged by the operator. + * + * @param opNamespace namespace in which the operator is running + * @param domainNamespace namespace in which the domain exists + * @param domainUid UID of the domain + * @param reason event to check for Created, Changed, deleted, processing etc + * @param type type of event, Normal of Warning + * @param timestamp the timestamp after which to see events + */ + public static void checkEvent( + String opNamespace, String domainNamespace, String domainUid, + String reason, String type, OffsetDateTime timestamp) { + withStandardRetryPolicy + .conditionEvaluationListener(condition -> + getLogger().info("Waiting for domain event {0} to be logged in namespace {1} " + + "(elapsed time {2}ms, remaining time {3}ms)", + reason, + domainNamespace, + condition.getElapsedTimeInMS(), + condition.getRemainingTimeInMS())) + .until(checkDomainEvent(opNamespace, domainNamespace, domainUid, reason, type, timestamp)); + } + + /** + * Delete a domain in the specified namespace. + * @param domainNS the namespace in which the domain exists + * @param domainUid domain uid + */ + public static void deleteDomainResource(String domainNS, String domainUid) { + //clean up domain resources in namespace and set namespace to label , managed by operator + getLogger().info("deleting domain custom resource {0}", domainUid); + assertTrue(deleteDomainCustomResource(domainUid, domainNS)); + + // wait until domain was deleted + withStandardRetryPolicy + .conditionEvaluationListener( + condition -> getLogger().info("Waiting for domain {0} to be deleted in namespace {1} " + + "(elapsed time {2}ms, remaining time {3}ms)", + domainUid, + domainNS, + condition.getElapsedTimeInMS(), + condition.getRemainingTimeInMS())) + .until(domainDoesNotExist(domainUid, DOMAIN_VERSION, domainNS)); + } + + private static Callable podLogContainsString(String namespace, String podName, String expectedString) { + return () -> { + String podLog; + try { + podLog = getPodLog(podName, namespace); + getLogger().info("pod log for pod {0} in namespace {1} : {2}", podName, namespace, podLog); + } catch (ApiException apiEx) { + getLogger().severe("got ApiException while getting pod log: ", apiEx); + return false; + } + + return podLog.contains(expectedString); + }; + } + + /** + * Wait and check the pod log contains the expected string. + * @param namespace the namespace in which the pod exists + * @param podName the pod to get the log + * @param expectedString the expected string to check in the pod log + */ + public static void checkPodLogContainsString(String namespace, String podName, String expectedString) { + + getLogger().info("Wait for string {0} existing in pod {1} in namespace {2}", expectedString, podName, namespace); + withStandardRetryPolicy + .conditionEvaluationListener( + condition -> getLogger().info("Waiting for string {0} existing in pod {1} in namespace {2} " + + "(elapsed time {3}ms, remaining time {4}ms)", + expectedString, + podName, + namespace, + condition.getElapsedTimeInMS(), + condition.getRemainingTimeInMS())) + .until(assertDoesNotThrow(() -> podLogContainsString(namespace, podName, expectedString), + "podLogContainsString failed with IOException, ApiException or InterruptedException")); + } + + /** + * Check the domain event contains the expected error msg. + * + * @param opNamespace namespace in which the operator is running + * @param domainNamespace namespace in which the domain exists + * @param domainUid UID of the domain + * @param reason event to check for Created, Changed, deleted, processing etc + * @param type type of event, Normal of Warning + * @param timestamp the timestamp after which to see events + * @param expectedMsg the expected message in the domain event message + */ + public static void checkDomainEventContainsExpectedMsg(String opNamespace, + String domainNamespace, + String domainUid, + String reason, + String type, + OffsetDateTime timestamp, + String expectedMsg) { + checkEvent(opNamespace, domainNamespace, domainUid, reason, type, timestamp); + CoreV1Event event = + getEvent(opNamespace, domainNamespace, domainUid, reason, type, timestamp); + if (event != null && event.getMessage() != null) { + assertTrue(event.getMessage().contains(expectedMsg), + String.format("The event message does not contain the expected msg %s", expectedMsg)); + } else { + fail("event is null or event message is null"); + } + } } diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/K8sEvents.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/K8sEvents.java index 2f22ce3f35f..eb33b62c96b 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/K8sEvents.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/K8sEvents.java @@ -159,7 +159,7 @@ public static boolean domainEventExists( } } } catch (ApiException ex) { - Logger.getLogger(ItKubernetesEvents.class.getName()).log(Level.SEVERE, null, ex); + logger.log(Level.SEVERE, null, ex); } return false; } diff --git a/integration-tests/src/test/resources/auxiliaryimage/Dockerfile b/integration-tests/src/test/resources/auxiliaryimage/Dockerfile index 53adfa77e9e..b0d28852f81 100644 --- a/integration-tests/src/test/resources/auxiliaryimage/Dockerfile +++ b/integration-tests/src/test/resources/auxiliaryimage/Dockerfile @@ -17,7 +17,7 @@ # FROM busybox -ARG AUXILIARY_IMAGE_PATH=/common +ARG AUXILIARY_IMAGE_PATH=/auxiliary ARG USER=oracle ARG USERID=1000 ARG GROUP=root