From afae77c119358905daca70098b43265b6aed5c43 Mon Sep 17 00:00:00 2001 From: huizhao Date: Wed, 28 Jul 2021 04:56:37 +0000 Subject: [PATCH 1/2] Added Integration Test to test cluster scoped auxiliary Image --- .../ItMiiAuxiliaryImageCluster.java | 398 ++++++++++++++++++ .../kubernetes/utils/CommonMiiTestUtils.java | 294 +++++++++++++ .../model-auxiliaryimage-cluster.yaml | 29 ++ 3 files changed, 721 insertions(+) create mode 100644 integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImageCluster.java create mode 100644 integration-tests/src/test/resources/wdt-models/model-auxiliaryimage-cluster.yaml diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImageCluster.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImageCluster.java new file mode 100644 index 00000000000..a436686c5b1 --- /dev/null +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImageCluster.java @@ -0,0 +1,398 @@ +// Copyright (c) 2021, Oracle and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.weblogic.kubernetes; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.List; +import java.util.Map; + +import oracle.weblogic.domain.Domain; +import oracle.weblogic.kubernetes.actions.impl.primitive.Command; +import oracle.weblogic.kubernetes.actions.impl.primitive.CommandParams; +import oracle.weblogic.kubernetes.annotations.IntegrationTest; +import oracle.weblogic.kubernetes.annotations.Namespaces; +import oracle.weblogic.kubernetes.logging.LoggingFacade; +import oracle.weblogic.kubernetes.utils.ExecResult; +import org.apache.commons.io.FileUtils; +import org.awaitility.core.ConditionFactory; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.SECONDS; +import static oracle.weblogic.kubernetes.TestConstants.ADMIN_PASSWORD_DEFAULT; +import static oracle.weblogic.kubernetes.TestConstants.ADMIN_USERNAME_DEFAULT; +import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_IMAGES_REPO; +import static oracle.weblogic.kubernetes.TestConstants.MII_AUXILIARY_IMAGE_NAME; +import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_TAG; +import static oracle.weblogic.kubernetes.TestConstants.OCIR_SECRET_NAME; +import static oracle.weblogic.kubernetes.TestConstants.RESULTS_ROOT; +import static oracle.weblogic.kubernetes.TestConstants.WEBLOGIC_IMAGE_NAME; +import static oracle.weblogic.kubernetes.TestConstants.WEBLOGIC_IMAGE_TAG; +import static oracle.weblogic.kubernetes.actions.ActionConstants.MODEL_DIR; +import static oracle.weblogic.kubernetes.actions.ActionConstants.RESOURCE_DIR; +import static oracle.weblogic.kubernetes.actions.TestActions.dockerPush; +import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort; +import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createDomainResourceWithAuxiliaryImageClusterScope; +import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.patchDomainClusterWithAuxImageAndVerify; +import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.readFilesInPod; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkSystemResourceConfiguration; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDomainAndVerify; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createOcirRepoSecret; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createSecretWithUsernamePassword; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getExternalServicePodName; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyOperator; +import static oracle.weblogic.kubernetes.utils.FileUtils.unzipWDTInstallationFile; +import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; +import static org.awaitility.Awaitility.with; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@DisplayName("Test to create model in image domain using auxiliary image containing the cluster configuration") +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@IntegrationTest +public class ItMiiAuxiliaryImageCluster { + + private static String opNamespace = null; + private static String domainNamespace = null; + private static LoggingFacade logger = null; + private static int auxiliaryImageNumberIndex = 1; + private final String domainUid = "domain1"; + private final String adminServerPodName = domainUid + "-admin-server"; + private final String managedServerPrefix = domainUid + "-managed-server"; + private final String clusterName = "cluster-1"; + private final int replicaCount = 2; + private final int clusterIndex = 0; + private final String miiAuxiliaryImagePrefix = MII_AUXILIARY_IMAGE_NAME + ":" + MII_BASIC_IMAGE_TAG; + private final String auxiliaryImageVolumeName = "auxiliaryImageVolumeCluster"; + private final String auxiliaryImagePath = "/auxiliary"; + private final String customDir = "customdir"; + + ConditionFactory withStandardRetryPolicy + = with().pollDelay(0, SECONDS) + .and().with().pollInterval(10, SECONDS) + .atMost(30, MINUTES).await(); + + /** + * Install Operator. + * @param namespaces list of namespaces created by the IntegrationTestWatcher by the + * JUnit engine parameter resolution mechanism + */ + @BeforeAll + public static void initAll(@Namespaces(2) List namespaces) { + logger = getLogger(); + // get a new unique opNamespace + logger.info("Creating unique namespace for Operator"); + assertNotNull(namespaces.get(0), "Namespace list is null"); + opNamespace = namespaces.get(0); + + logger.info("Creating unique namespace for Domain1"); + assertNotNull(namespaces.get(1), "Namespace list is null"); + domainNamespace = namespaces.get(1); + + // install and verify operator + installAndVerifyOperator(opNamespace, domainNamespace); + } + + /** + * Create a domain using multiple auxiliary images. One auxiliary image containing the domain configuration + * and other two auxiliary images contain the cluster configuration, verify the domain is running and + * files in cluster scope image only copied to WLS server within the cluster. + */ + @Test + @Order(1) + @DisplayName("Test to create domain using multiple auxiliary images containing the doamin and cluster configuration") + public void testCreateDomainUsingAuxiliaryImagesWClusterConfig() { + final List auxiliaryImageDomainScopeNames = List.of(miiAuxiliaryImagePrefix + "1"); + final List auxiliaryImageClusterScopeNames = + List.of(miiAuxiliaryImagePrefix + "2", miiAuxiliaryImagePrefix + "3"); + auxiliaryImageNumberIndex += 3; + + // Create the repo secret to pull the image + // this secret is used only for non-kind cluster + createOcirRepoSecret(domainNamespace); + + // create secret for admin credentials + logger.info("Create secret for admin credentials"); + String adminSecretName = "weblogic-credentials"; + createSecretWithUsernamePassword(adminSecretName, domainNamespace, + ADMIN_USERNAME_DEFAULT, ADMIN_PASSWORD_DEFAULT); + + // create encryption secret + logger.info("Create encryption secret"); + String encryptionSecretName = "encryptionsecret"; + createSecretWithUsernamePassword(encryptionSecretName, domainNamespace, + "weblogicenc", "weblogicenc"); + + // create stage dir for domain scope auxiliary image with auxiliaryImageDomainScopeNames + Path multipleAiPath1 = Paths.get(RESULTS_ROOT, "multipleauxiliaryimage1"); + // create models dir and copy model, archive files if any for auxiliaryImageDomainScopeNames + Path modelsPath = Paths.get(multipleAiPath1.toString(), "models"); + logger.info("Create models dir {0} and copy model file {1}", + multipleAiPath1.toString(), Paths.get(MODEL_DIR, "model-auxiliaryimage-cluster.yaml").toString()); + assertDoesNotThrow(() -> FileUtils.deleteDirectory(modelsPath.toFile())); + assertDoesNotThrow(() -> Files.createDirectories(modelsPath)); + assertDoesNotThrow(() -> Files.copy( + Paths.get(MODEL_DIR, "model-auxiliaryimage-cluster.yaml"), + Paths.get(modelsPath.toString(), "model-auxiliaryimage-cluster.yaml"), + StandardCopyOption.REPLACE_EXISTING)); + + // unzip WDT installation file into work dir + unzipWDTInstallationFile(multipleAiPath1.toString()); + + // create auxiliaryImageDomainScopeNames with model and wdt installation files + logger.info("Create auxiliary image: {0}", auxiliaryImageDomainScopeNames.get(0)); + createAuxiliaryImage(multipleAiPath1.toString(), + Paths.get(RESOURCE_DIR, "auxiliaryimage", "Dockerfile").toString(), + auxiliaryImageDomainScopeNames.get(0)); + + // create stage and customdir dir for cluster scope auxiliary images with auxiliaryImageClusterScopeNames + Path multipleAiPath2 = Paths.get(RESULTS_ROOT, "multipleauxiliaryimage2"); + Path customDirPath = Paths.get(multipleAiPath2.toString(), customDir); + logger.info("Create a custom dir {0} and copy two text files {1} and {2}", + customDirPath.toString(), "clusterai-1.txt", "clusterai-2.txt"); + assertDoesNotThrow(() -> FileUtils.deleteDirectory(customDirPath.toFile())); + assertDoesNotThrow(() -> Files.createDirectories(customDirPath)); + + // create and copy two plain text files + Map fileNames = + Map.of(Paths.get(customDirPath.toString(), "clusterai-1.txt"), "file 1 in customdir", + Paths.get(customDirPath.toString(), "clusterai-2.txt"), "file 2 in customdir"); + fileNames.forEach((filePath, fileContent) -> { + assertDoesNotThrow(() -> Files.write(filePath, fileContent.getBytes()), + "Can't write to file " + filePath); + }); + + // create an auxiliary image containing each plain text file, respectively + auxiliaryImageClusterScopeNames.stream().forEach( + imageName -> { + logger.info("Create auxiliary image: {0}", imageName); + createAuxiliaryImage(multipleAiPath2.toString(), + Paths.get(RESOURCE_DIR, "auxiliaryimage", "Dockerfile").toString(), imageName); + } + ); + + // create domain custom resource using 3 auxiliary images + logger.info("Creating domain custom resource with domainUid {0} and auxiliary images {1} {2}", + domainUid, auxiliaryImageDomainScopeNames.toString(), auxiliaryImageClusterScopeNames.toString()); + Domain domainCR = createDomainResourceWithAuxiliaryImageClusterScope(domainUid, domainNamespace, + WEBLOGIC_IMAGE_NAME + ":" + WEBLOGIC_IMAGE_TAG, adminSecretName, OCIR_SECRET_NAME, + encryptionSecretName, replicaCount, clusterName, + Map.of(auxiliaryImagePath, List.of(auxiliaryImageVolumeName)), + auxiliaryImageDomainScopeNames, auxiliaryImageClusterScopeNames); + + // create domain and verify its running + logger.info("Creating domain {0} with auxiliary images {1} {2} in namespace {3}", + domainUid, auxiliaryImageDomainScopeNames.toString(), + auxiliaryImageClusterScopeNames.toString(), domainNamespace); + createDomainAndVerify(domainUid, domainCR, domainNamespace, + adminServerPodName, managedServerPrefix, replicaCount); + + // verify that plain text files only copied to servers in the cluster but not copied to admin server + fileNames.forEach((filePath, fileContent) -> { + verifyFileInPod(Paths.get(auxiliaryImagePath, customDir, filePath.getFileName().toString()).toString(), + fileContent); + }); + } + + /** + * Patch a domain using auxiliary image to add a new cluster configuration, + * verify the domain is patched and rolling restarted and + * file in cluster scope image only copied to WLS server within the cluster. + */ + @Test + @Order(2) + @DisplayName("Patch a domain using auxiliary image to add a new cluster configuration") + public void testPatchDomainToAddClusterConfigUsingAuxiliaryImage() { + final int addAuxImageLoc = 2; + final String miiAuxiliaryImage = miiAuxiliaryImagePrefix + auxiliaryImageNumberIndex; + auxiliaryImageNumberIndex += 1; + + // create stage dir for cluster scope auxiliary images with miiAuxiliaryImage + Path customAiPath = Paths.get(RESULTS_ROOT, "multipleauxiliaryimage_add"); + Path customDirPath = Paths.get(customAiPath.toString(), customDir); + assertDoesNotThrow(() -> FileUtils.deleteDirectory(customDirPath.toFile())); + assertDoesNotThrow(() -> Files.createDirectories(customDirPath)); + logger.info("Create a custom dir {0} and copy text file {1}", + customDirPath.toString(), "clusteraiAdd.txt"); + + // create a plain text file + Path fileInCustomDir = Paths.get(customDirPath.toString(), "clusteraiAdd.txt"); + String fileContent = "patch the domain to add"; + assertDoesNotThrow(() -> Files.write(fileInCustomDir, fileContent.getBytes()), + "Can't write to file " + fileInCustomDir); + + // create image containing plain text file clusteraiAdd.txt + logger.info("Create auxiliary image: {0}", miiAuxiliaryImage); + createAuxiliaryImage(customAiPath.toString(), + Paths.get(RESOURCE_DIR, "auxiliaryimage", "Dockerfile").toString(), miiAuxiliaryImage); + + // push image to repo for multi node cluster + if (!DOMAIN_IMAGES_REPO.isEmpty()) { + logger.info("docker push image {0} to registry {1}", miiAuxiliaryImage, DOMAIN_IMAGES_REPO); + assertTrue(dockerPush(miiAuxiliaryImage), String.format("docker push failed for image %s", miiAuxiliaryImage)); + } + + // patch a domain using auxiliary image to add a new cluster configuration + logger.info("Patch domain to add a new cluster config to cluster {0}", clusterName); + patchDomainClusterWithAuxImageAndVerify(domainUid, domainNamespace, managedServerPrefix, replicaCount, + clusterIndex, auxiliaryImageVolumeName, miiAuxiliaryImage, addAuxImageLoc, "add"); + + // verify that plain text file only copied to servers in the cluster but not copied to admin server + logger.info("Verify that plain text file only copied to servers in the cluster but not copied to admin server"); + verifyFileInPod(Paths.get(auxiliaryImagePath, customDir, fileInCustomDir.getFileName().toString()).toString(), + fileContent); + } + + /** + * Patch a domain using auxiliary image to replace an existing cluster configuration, + * verify the domain is patched and rolling restarted, + * files in cluster scope image only copied to WLS server within the cluster. + */ + @Test + @Order(3) + @DisplayName("Patch a domain using auxiliary image to replace an existing cluster configuration") + public void testPatchDomainToReplaceClusterConfigUsingAuxiliaryImage() { + final int replaceAuxImageLoc = 2; + final String miiAuxiliaryImage = miiAuxiliaryImagePrefix + auxiliaryImageNumberIndex; + auxiliaryImageNumberIndex += 1; + + // create stage and customdir dir for cluster scope auxiliary images with miiAuxiliaryImage + Path customAiPath = Paths.get(RESULTS_ROOT, "multipleauxiliaryimage_replace"); + Path customDirPath = Paths.get(customAiPath.toString(), customDir); + assertDoesNotThrow(() -> FileUtils.deleteDirectory(customDirPath.toFile())); + assertDoesNotThrow(() -> Files.createDirectories(customDirPath)); + logger.info("Create a custom dir {0} and copy text file {1}", + customDirPath.toString(), "clusteraiReplace.txt"); + + // create a plain text file + Path fileInCustomDir = Paths.get(customDirPath.toString(), "clusteraiReplace.txt"); + String fileContent = "patch the domain to replace"; + assertDoesNotThrow(() -> Files.write(fileInCustomDir, fileContent.getBytes()), + "Can't write to file " + fileInCustomDir); + + // create image with plain text file clusteraiReplace.txt + logger.info("Create auxiliary image: {0}", miiAuxiliaryImage); + createAuxiliaryImage(customAiPath.toString(), + Paths.get(RESOURCE_DIR, "auxiliaryimage", "Dockerfile").toString(), miiAuxiliaryImage); + + // push image to repo for multi node cluster + if (!DOMAIN_IMAGES_REPO.isEmpty()) { + logger.info("docker push image {0} to registry {1}", miiAuxiliaryImage, DOMAIN_IMAGES_REPO); + assertTrue(dockerPush(miiAuxiliaryImage), String.format("docker push failed for image %s", miiAuxiliaryImage)); + } + + // patch a domain using auxiliary image to replace an existing cluster configuration + logger.info("Patch domain to replace an existing cluster config to cluster {0}", clusterName); + patchDomainClusterWithAuxImageAndVerify(domainUid, domainNamespace, managedServerPrefix, replicaCount, + clusterIndex, auxiliaryImageVolumeName, miiAuxiliaryImage, replaceAuxImageLoc, "replace"); + + // verify that plain text file only copied to servers in the cluster but not copied to admin server + logger.info("Verify that plain text file only copied to servers in the cluster but not copied to admin server"); + verifyFileInPod(Paths.get(auxiliaryImagePath, customDir, fileInCustomDir.getFileName().toString()).toString(), + fileContent); + } + + /** + * A negative test to verify that model files in an auxiliary images at the cluster scope will be ignored. + * The test creates an auxiliary image containing JMS model files and use it + * to config the cluster scope configuration. + * Verify that the JMS system resource config using adminServiceNodePort can not be found. + */ + @Test + @Order(4) + @DisplayName("Verify that model files in an auxiliary images at the cluster scope are ignored") + public void testPatchDomainToAddModelsToClusterConfigIgnored() { + final int addAuxImageLoc = 3; + final String miiAuxiliaryImage = miiAuxiliaryImagePrefix + auxiliaryImageNumberIndex; + auxiliaryImageNumberIndex += 1; + + // stage dir for auxiliary image containing the cluster config and copy JMS model file + Path auxiliaryImagePath = Paths.get(RESULTS_ROOT, "auxiliaryimage_modelcluster"); + Path modelsPath = Paths.get(auxiliaryImagePath.toString(), "models"); + logger.info("Create models dir {0} and copy model file {1}", + modelsPath.toString(), Paths.get(MODEL_DIR, "/model.jms2.yaml").toString()); + + assertDoesNotThrow(() -> FileUtils.deleteDirectory(modelsPath.toFile())); + assertDoesNotThrow(() -> Files.createDirectories(modelsPath)); + assertDoesNotThrow(() -> Files.copy( + Paths.get(MODEL_DIR, "/model.jms2.yaml"), + Paths.get(modelsPath.toString(), "/model.jms2.yaml"), + StandardCopyOption.REPLACE_EXISTING)); + + // create image with JMS model file + logger.info("Create auxiliary image: {0}", miiAuxiliaryImage); + createAuxiliaryImage(auxiliaryImagePath.toString(), + Paths.get(RESOURCE_DIR, "auxiliaryimage", "Dockerfile").toString(), miiAuxiliaryImage); + + // push image to repo for multi node cluster + if (!DOMAIN_IMAGES_REPO.isEmpty()) { + logger.info("docker push image {0} to registry {1}", miiAuxiliaryImage, DOMAIN_IMAGES_REPO); + assertTrue(dockerPush(miiAuxiliaryImage), String.format("docker push failed for image %s", miiAuxiliaryImage)); + } + + // patch a domain using the auxiliary image containing to config cluster scope configuration + logger.info("Patch domain to add a new cluster config to cluster {0}", clusterName); + patchDomainClusterWithAuxImageAndVerify(domainUid, domainNamespace, managedServerPrefix, replicaCount, + clusterIndex, auxiliaryImageVolumeName, miiAuxiliaryImage, addAuxImageLoc, "add"); + + // verify that the JMS system resource config using adminServiceNodePort can not be found + logger.info("Verify that the JMSSystemResource configuration doesn't exist"); + int adminServiceNodePort = + getServiceNodePort(domainNamespace, getExternalServicePodName(adminServerPodName), "default"); + assertNotEquals(-1, adminServiceNodePort, "admin server default node port is not valid"); + assertFalse(checkSystemResourceConfiguration(adminServiceNodePort, "JMSSystemResources", + "TestClusterJmsModule2", "200"), "JMSSystemResources found"); + logger.info("The JMSSystemResource configuration is not found"); + } + + private void createAuxiliaryImage(String stageDirPath, String dockerFileLocation, String auxiliaryImage) { + String cmdToExecute = String.format("cd %s && docker build -f %s %s -t %s .", + stageDirPath, dockerFileLocation, + "--build-arg AUXILIARY_IMAGE_PATH=" + auxiliaryImagePath, auxiliaryImage); + assertTrue(new Command() + .withParams(new CommandParams() + .command(cmdToExecute)) + .execute(), String.format("Failed to execute", cmdToExecute)); + + // push miiAuxiliaryImage to repo for multi node cluster + if (!DOMAIN_IMAGES_REPO.isEmpty()) { + logger.info("docker push image {0} to registry {1}", auxiliaryImage, DOMAIN_IMAGES_REPO); + assertTrue(dockerPush(auxiliaryImage), String.format("docker push failed for image %s", auxiliaryImage)); + } + } + + private void verifyFileInPod(String fileName, String fileContent) { + // verify that the file is not copied to admin server that is not a server in the cluster + ExecResult result = readFilesInPod(domainNamespace, adminServerPodName, fileName); + + logger.info("readFilesInPod returned: {0}", result.toString()); + assertFalse(result.exitValue() == 0, String.format("Failed to read file %s. Error is: %s", + fileName, result.stderr())); + assertTrue(result.toString().contains("No such file or directory"), + String.format("File %s should not exists in the admin pod", fileName)); + + // verify that the file is copied to managed server that is a server in the cluster + for (int i = 1; i <= replicaCount; i++) { + result = readFilesInPod(domainNamespace, managedServerPrefix + i, fileName); + + logger.info("readFilesInPod returned: {0}", result.toString()); + assertTrue(result.exitValue() == 0, String.format("Failed to read file %s. Error is: %s", + fileName, result.stderr())); + assertTrue(result.stdout().contains(fileContent), + String.format("The content %s read from file %s is not same as given one %s in managed server pod %s", + result.stdout(), fileName, fileContent, managedServerPrefix + i)); + } + } +} diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java index f10b280def8..fcf669cec1d 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java @@ -12,6 +12,7 @@ import java.util.Map; import java.util.Set; +import io.kubernetes.client.custom.V1Patch; import io.kubernetes.client.openapi.models.V1Container; import io.kubernetes.client.openapi.models.V1EnvVar; import io.kubernetes.client.openapi.models.V1Job; @@ -61,11 +62,14 @@ import static oracle.weblogic.kubernetes.actions.TestActions.createDomainCustomResource; import static oracle.weblogic.kubernetes.actions.TestActions.createSecret; import static oracle.weblogic.kubernetes.actions.TestActions.deleteConfigMap; +import static oracle.weblogic.kubernetes.actions.TestActions.getDomainCustomResource; import static oracle.weblogic.kubernetes.actions.TestActions.getJob; import static oracle.weblogic.kubernetes.actions.TestActions.getPod; +import static oracle.weblogic.kubernetes.actions.TestActions.getPodCreationTimestamp; import static oracle.weblogic.kubernetes.actions.TestActions.getPodLog; import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort; import static oracle.weblogic.kubernetes.actions.TestActions.listPods; +import static oracle.weblogic.kubernetes.actions.TestActions.patchDomainCustomResource; import static oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes.listConfigMaps; import static oracle.weblogic.kubernetes.assertions.TestAssertions.podIntrospectVersionUpdated; import static oracle.weblogic.kubernetes.assertions.TestAssertions.verifyRollingRestartOccurred; @@ -90,6 +94,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -344,6 +349,181 @@ public static Domain createDomainResourceWithAuxiliaryImage( return domainCR; } + /** + * Create a domain object for a Kubernetes domain custom resource using the basic WLS image + * and MII auxiliary images containing the doamin or/and cluster configuration. + * + * @param domainResourceName name of the domain resource + * @param domNamespace Kubernetes namespace that the domain is hosted + * @param baseImageName name of the base image to use + * @param adminSecretName name of the new WebLogic admin credentials secret + * @param repoSecretName name of the secret for pulling the WebLogic image + * @param encryptionSecretName name of the secret used to encrypt the models + * @param replicaCount number of managed servers to start + * @param clusterNames a list of the cluster name to add auxiliary image in domain + * @param auxiliaryImagePathVolume a map of auxiliary image path, parent location for Model in Image model + * and WDT installation files as the key and a list of auxiliary image volume names + * as the values for the key + * @param auxiliaryImageDomainScopeNames a list of image names including tags, image contains the domain model, + * application archive if any and WDT installation files + * @param auxiliaryImageClusterScopeNames a list of images containing the files to + * config cluster scope auxiliary image + * @return domain object of the domain resource + */ + public static Domain createDomainResourceWithAuxiliaryImageClusterScope( + String domainResourceName, + String domNamespace, + String baseImageName, + String adminSecretName, + String repoSecretName, + String encryptionSecretName, + int replicaCount, + List clusterNames, + Map> auxiliaryImagePathVolume, + List auxiliaryImageDomainScopeNames, + List auxiliaryImageClusterScopeNames) { + + Domain domainCR = null; + int i = 0; + + for (String clusterName : clusterNames) { + domainCR = + createDomainResourceWithAuxiliaryImageClusterScope(domainResourceName, + domNamespace, + baseImageName, + adminSecretName, + repoSecretName, + encryptionSecretName, + replicaCount, + clusterName, + auxiliaryImagePathVolume, + auxiliaryImageDomainScopeNames, + auxiliaryImageClusterScopeNames, + domainCR); + } + + return domainCR; + } + + /** + * Create a domain object for a Kubernetes domain custom resource using the basic WLS image + * and MII auxiliary images containing the doamin or/and cluster configuration. + * + * @param domainResourceName name of the domain resource + * @param domNamespace Kubernetes namespace that the domain is hosted + * @param baseImageName name of the base image to use + * @param adminSecretName name of the new WebLogic admin credentials secret + * @param repoSecretName name of the secret for pulling the WebLogic image + * @param encryptionSecretName name of the secret used to encrypt the models + * @param replicaCount number of managed servers to start + * @param clusterName name of the cluster to add in domain + * @param auxiliaryImagePathVolume a map of auxiliary image path, parent location for Model in Image model + * and WDT installation files as the key and a list of auxiliary image volume names + * as the values for the key + * @param auxiliaryImageDomainScopeNames a list of image names including tags, image contains the domain model, + * application archive if any and WDT installation files + * @param auxiliaryImageClusterScopeNames a list of images containing the files to + * config cluster scope auxiliary image + * @return domain object of the domain resource + */ + public static Domain createDomainResourceWithAuxiliaryImageClusterScope( + String domainResourceName, + String domNamespace, + String baseImageName, + String adminSecretName, + String repoSecretName, + String encryptionSecretName, + int replicaCount, + String clusterName, + Map> auxiliaryImagePathVolume, + List auxiliaryImageDomainScopeNames, + List auxiliaryImageClusterScopeNames, + Domain... domainCRParam) { + + Domain domainCR = (domainCRParam != null && domainCRParam.length != 0) ? domainCRParam[0] : + CommonMiiTestUtils.createDomainResource(domainResourceName, domNamespace, + baseImageName, adminSecretName, repoSecretName, + encryptionSecretName, replicaCount, clusterName); + + auxiliaryImagePathVolume.forEach((auxiliaryImagePath, auxiliaryImageVolumes) -> { + System.out.println(auxiliaryImagePath + " - " + auxiliaryImageVolumes.toString()); + for (String auxiliaryImageVolumeName : auxiliaryImageVolumes) { + domainCR.spec().addAuxiliaryImageVolumesItem(new AuxiliaryImageVolume() + .mountPath(auxiliaryImagePath) + .name(auxiliaryImageVolumeName)); + domainCR.spec().configuration().model() + .withModelHome(auxiliaryImagePath + "/models") + .withWdtInstallHome(auxiliaryImagePath + "/weblogic-deploy"); + + appendDomainResourceDomainScopeAuxiliaryImage(domainCR, auxiliaryImageVolumeName, + auxiliaryImageDomainScopeNames); + appendDomainResourceClusterScopeAuxiliaryImage(domainCR, clusterName, + auxiliaryImageVolumeName, auxiliaryImageClusterScopeNames); + } + }); + + return domainCR; + } + + /** + * Append domain scope config in a MII auxiliary image to domain object. + * + * @param domainCR domain object of the domain resource to add cluster scope auxiliary image configurations to + * @param auxiliaryImageVolumeName auxiliary image volume name + * @param auxiliaryImageDomainScopeNames a list of image names including tags, image contains the domain model, + * application archive if any and WDT installation files + * @return domain object of the domain resource + */ + public static Domain appendDomainResourceDomainScopeAuxiliaryImage( + Domain domainCR, + String auxiliaryImageVolumeName, + List auxiliaryImageDomainScopeNames) { + + for (String auxiliaryImageName: auxiliaryImageDomainScopeNames) { + domainCR.spec().serverPod() + .addAuxiliaryImagesItem(new AuxiliaryImage() + .image(auxiliaryImageName) + .volume(auxiliaryImageVolumeName) + .imagePullPolicy("IfNotPresent")); + } + + return domainCR; + } + + /** + * Append cluster scope config in a MII auxiliary image to domain object. + * + * @param domainCR domain object of the domain resource to add cluster scope auxiliary image configurations to + * @param clusterName name of the cluster to add in domain + * @param auxiliaryImageVolumeName auxiliary image volume name + * @param auxiliaryImageClusterScopeNames a list of image containing the files to + * config cluster scope auxiliary images + * @return domain object of the domain resource + */ + public static Domain appendDomainResourceClusterScopeAuxiliaryImage( + Domain domainCR, + String clusterName, + String auxiliaryImageVolumeName, + List auxiliaryImageClusterScopeNames) { + + for (String auxiliaryImageName: auxiliaryImageClusterScopeNames) { + domainCR.spec().getClusters() + .stream() + .forEach( + cluster -> { + if (cluster.getClusterName().equals(clusterName)) { + cluster.serverPod().addAuxiliaryImagesItem(new AuxiliaryImage() + .image(auxiliaryImageName) + .volume(auxiliaryImageVolumeName) + .imagePullPolicy("IfNotPresent")); + } + } + ); + } + + return domainCR; + } + /** * Create a domain object for a Kubernetes domain custom resource using the basic model-in-image * image. @@ -1003,4 +1183,118 @@ public static void verifyUpdateWebLogicCredential(String domainNamespace, String getLogger().info("Domain {0} in namespace {1} is fully started after changing WebLogic credentials secret", domainUid, domainNamespace); } + + + /** + * Patch the domain CRD with a new auxiliary image to add new or replace existing + * auxiliary images at cluster scope. Verify the server pods in cluster are rolling + * restarted and back to ready state. + * @param domainNamespace namespace where the domain is + * @param managedServerPrefix prefix of the managed server + * @param replicaCount replica count of the domain + * @param clusterIndex index of cluster to add or replace the auxiliary image cluster config + * @param auxiliaryImageVolumeName auxiliary image volume name + * @param auxiliaryImageName image names containing the files to config cluster scope auxiliary image + * @param auxiliaryImageIndex location to add or replace the auxiliary image cluster config + * @param addOrReplace add or replace the auxiliary image cluster config + */ + public static void patchDomainClusterWithAuxImageAndVerify(String domainUid, + String domainNamespace, + String managedServerPrefix, + int replicaCount, + int clusterIndex, + String auxiliaryImageVolumeName, + String auxiliaryImageName, + int auxiliaryImageIndex, + String addOrReplace) { + + LoggingFacade logger = getLogger(); + + // create the map with server pods and their original creation timestamps + Map podsWithTimeStamps = new LinkedHashMap<>(); + for (int i = 1; i <= replicaCount; i++) { + String managedServerPodName = managedServerPrefix + i; + podsWithTimeStamps.put(managedServerPodName, + assertDoesNotThrow(() -> getPodCreationTimestamp(domainNamespace, "", managedServerPodName), + String.format("getPodCreationTimestamp failed with ApiException for pod %s in namespace %s", + managedServerPodName, domainNamespace))); + } + + // create patch string + StringBuffer patchStr = new StringBuffer("[") + .append("{\"op\": \"" + addOrReplace + "\",") + .append(" \"path\": \"/spec/clusters/") + .append(clusterIndex) + .append("/serverPod/auxiliaryImages/") + .append(auxiliaryImageIndex) + .append("\", ") + .append("\"value\": {\"image\": \"") + .append(auxiliaryImageName) + .append("\", ") + .append("\"imagePullPolicy\": \"IfNotPresent\", ") + .append("\"volume\": ") + .append("\"") + .append(auxiliaryImageVolumeName) + .append("\"}}]"); + + logger.info("Patch domain with auxiliary image patch string: " + patchStr); + + // patch the domain and verify + V1Patch patch = new V1Patch((patchStr).toString()); + boolean aiPatched = assertDoesNotThrow(() -> + patchDomainCustomResource(domainUid, domainNamespace, patch, "application/json-patch+json"), + "patchDomainClusterWithAuxiliaryImageAndVerify failed "); + assertTrue(aiPatched, "patchDomainClusterWithAuxiliaryImageAndVerify failed"); + + Domain domain1 = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, domainNamespace), + String.format("getDomainCustomResource failed with ApiException when tried to get domain %s in namespace %s", + domainUid, domainNamespace)); + assertNotNull(domain1, "Got null domain resource after patching"); + assertNotNull(domain1.getSpec().getClusters().get(clusterIndex).getServerPod().getAuxiliaryImages(), + domain1 + "/spec/serverPod/auxiliaryImages is null"); + + //verify that the domain is patched with new image + List auxiliaryImageListAf = + domain1.getSpec().getClusters().get(clusterIndex).getServerPod().getAuxiliaryImages(); + boolean doMainPatched = false; + for (AuxiliaryImage auxImage : auxiliaryImageListAf) { + if (auxImage.getImage().equals(auxiliaryImageName)) { + logger.info("Domain patched and cluster config {0} found", auxImage); + doMainPatched = true; + break; + } + } + assertTrue(doMainPatched, String.format("Image name %s should be patched", auxiliaryImageName)); + + // verify the server pods in cluster are rolling restarted and back to ready state + logger.info("Verifying rolling restart occurred for domain {0} in namespace {1}", + domainUid, domainNamespace); + assertTrue(verifyRollingRestartOccurred(podsWithTimeStamps, 1, domainNamespace), + String.format("Rolling restart failed for domain %s in namespace %s", domainUid, domainNamespace)); + } + + /** + * Read a file in a given pod. + * @param domainNamespace namespace where the domain is + * @param serverPodName WLS server pod name + * @param fileName file to read from + * @return ExecResult containing the content of the given file + */ + public static ExecResult readFilesInPod(String domainNamespace, + String serverPodName, + String fileName) { + LoggingFacade logger = getLogger(); + StringBuffer readFileCmd = new StringBuffer("kubectl exec -n ") + .append(domainNamespace) + .append(" ") + .append(serverPodName) + .append(" -- cat \"") + .append(fileName) + .append("\""); + logger.info("command to read file in pod {0} is: {1}", serverPodName, readFileCmd.toString()); + + ExecResult result = assertDoesNotThrow(() -> exec(readFileCmd.toString(), true)); + + return result; + } } diff --git a/integration-tests/src/test/resources/wdt-models/model-auxiliaryimage-cluster.yaml b/integration-tests/src/test/resources/wdt-models/model-auxiliaryimage-cluster.yaml new file mode 100644 index 00000000000..05ab65c12dc --- /dev/null +++ b/integration-tests/src/test/resources/wdt-models/model-auxiliaryimage-cluster.yaml @@ -0,0 +1,29 @@ +# Copyright (c) 2021, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +domainInfo: + AdminUserName: '@@SECRET:__weblogic-credentials__:username@@' + AdminPassword: '@@SECRET:__weblogic-credentials__:password@@' + ServerStartMode: 'prod' + +topology: + Name: "domain1" + AdminServerName: "admin-server" + Cluster: + "cluster-1": + DynamicServers: + ServerTemplate: "cluster-1-template" + ServerNamePrefix: "managed-server" + DynamicClusterSize: 5 + MaxDynamicClusterSize: 5 + CalculatedListenPorts: false + Server: + "admin-server": + ListenPort: 7001 + ServerTemplate: + "cluster-1-template": + Cluster: "cluster-1" + ListenPort : 8001 + WebServer: + WebServerLog: + BufferSizeKb: 1 \ No newline at end of file From 5b3a74bc4aa97049a84d997d79dfe4271e6f5e30 Mon Sep 17 00:00:00 2001 From: huizhao Date: Fri, 30 Jul 2021 21:03:53 +0000 Subject: [PATCH 2/2] Changes based on comments --- .../kubernetes/utils/CommonMiiTestUtils.java | 66 ++----------------- 1 file changed, 4 insertions(+), 62 deletions(-) diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java index fcf669cec1d..f91d311cacb 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java @@ -349,62 +349,6 @@ public static Domain createDomainResourceWithAuxiliaryImage( return domainCR; } - /** - * Create a domain object for a Kubernetes domain custom resource using the basic WLS image - * and MII auxiliary images containing the doamin or/and cluster configuration. - * - * @param domainResourceName name of the domain resource - * @param domNamespace Kubernetes namespace that the domain is hosted - * @param baseImageName name of the base image to use - * @param adminSecretName name of the new WebLogic admin credentials secret - * @param repoSecretName name of the secret for pulling the WebLogic image - * @param encryptionSecretName name of the secret used to encrypt the models - * @param replicaCount number of managed servers to start - * @param clusterNames a list of the cluster name to add auxiliary image in domain - * @param auxiliaryImagePathVolume a map of auxiliary image path, parent location for Model in Image model - * and WDT installation files as the key and a list of auxiliary image volume names - * as the values for the key - * @param auxiliaryImageDomainScopeNames a list of image names including tags, image contains the domain model, - * application archive if any and WDT installation files - * @param auxiliaryImageClusterScopeNames a list of images containing the files to - * config cluster scope auxiliary image - * @return domain object of the domain resource - */ - public static Domain createDomainResourceWithAuxiliaryImageClusterScope( - String domainResourceName, - String domNamespace, - String baseImageName, - String adminSecretName, - String repoSecretName, - String encryptionSecretName, - int replicaCount, - List clusterNames, - Map> auxiliaryImagePathVolume, - List auxiliaryImageDomainScopeNames, - List auxiliaryImageClusterScopeNames) { - - Domain domainCR = null; - int i = 0; - - for (String clusterName : clusterNames) { - domainCR = - createDomainResourceWithAuxiliaryImageClusterScope(domainResourceName, - domNamespace, - baseImageName, - adminSecretName, - repoSecretName, - encryptionSecretName, - replicaCount, - clusterName, - auxiliaryImagePathVolume, - auxiliaryImageDomainScopeNames, - auxiliaryImageClusterScopeNames, - domainCR); - } - - return domainCR; - } - /** * Create a domain object for a Kubernetes domain custom resource using the basic WLS image * and MII auxiliary images containing the doamin or/and cluster configuration. @@ -437,13 +381,11 @@ public static Domain createDomainResourceWithAuxiliaryImageClusterScope( String clusterName, Map> auxiliaryImagePathVolume, List auxiliaryImageDomainScopeNames, - List auxiliaryImageClusterScopeNames, - Domain... domainCRParam) { + List auxiliaryImageClusterScopeNames) { - Domain domainCR = (domainCRParam != null && domainCRParam.length != 0) ? domainCRParam[0] : - CommonMiiTestUtils.createDomainResource(domainResourceName, domNamespace, - baseImageName, adminSecretName, repoSecretName, - encryptionSecretName, replicaCount, clusterName); + Domain domainCR = CommonMiiTestUtils.createDomainResource(domainResourceName, + domNamespace, baseImageName, adminSecretName, repoSecretName, + encryptionSecretName, replicaCount, clusterName); auxiliaryImagePathVolume.forEach((auxiliaryImagePath, auxiliaryImageVolumes) -> { System.out.println(auxiliaryImagePath + " - " + auxiliaryImageVolumes.toString());