From 9abcc9291b7629566d397b47642f94d4a60a9b1c Mon Sep 17 00:00:00 2001 From: xiancao Date: Wed, 12 Aug 2020 23:30:13 +0000 Subject: [PATCH 01/15] initial commit --- .../kubernetes/actions/impl/Apache.java | 176 ++++++++++++++++++ .../kubernetes/actions/impl/ApacheParams.java | 112 +++++++++++ 2 files changed, 288 insertions(+) create mode 100644 new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/Apache.java create mode 100644 new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/ApacheParams.java diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/Apache.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/Apache.java new file mode 100644 index 00000000000..575bbe254e2 --- /dev/null +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/Apache.java @@ -0,0 +1,176 @@ +// Copyright (c) 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.weblogic.kubernetes.actions.impl; + +import io.kubernetes.client.custom.IntOrString; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.models.NetworkingV1beta1HTTPIngressPath; +import io.kubernetes.client.openapi.models.NetworkingV1beta1HTTPIngressRuleValue; +import io.kubernetes.client.openapi.models.NetworkingV1beta1Ingress; +import io.kubernetes.client.openapi.models.NetworkingV1beta1IngressBackend; +import io.kubernetes.client.openapi.models.NetworkingV1beta1IngressList; +import io.kubernetes.client.openapi.models.NetworkingV1beta1IngressRule; +import io.kubernetes.client.openapi.models.NetworkingV1beta1IngressSpec; +import io.kubernetes.client.openapi.models.V1ObjectMeta; +import oracle.weblogic.kubernetes.actions.impl.primitive.Helm; +import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams; +import oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; + +/** + * Utility class for Apache ingress controller. + */ +public class Apache { + + private static final String INGRESS_API_VERSION = "networking.k8s.io/v1beta1"; + private static final String INGRESS_KIND = "Ingress"; + private static final String INGRESS_NGINX_CLASS = "nginx"; + + /** + * Install Apache Helm chart. + * + * @param params the parameters to Helm install command such as release name, namespace, repo url or chart dir, + * chart name and chart values + * @return true on success, false otherwise + */ + public static boolean install(ApacheParams params) { + return Helm.install(params.getHelmParams(), params.getValues()); + } + + /** + * Upgrade Apache Helm release. + * + * @param params the parameters to Helm upgrade command such as release name, namespace and chart values to override + * @return true on success, false otherwise + */ + public static boolean upgrade(ApacheParams params) { + return Helm.upgrade(params.getHelmParams(), params.getValues()); + } + + /** + * Uninstall Apache Helm release. + * + * @param params the parameters to Helm uninstall command such as release name and namespace + * @return true on success, false otherwise + */ + public static boolean uninstall(HelmParams params) { + return Helm.uninstall(params); + } + + /** + * Create an ingress for the WebLogic domain with domainUid in the specified domain namespace. + * The ingress host is set to 'domainUid.clusterName.test'. + * + * @param ingressName name of the ingress to be created + * @param domainNamespace the WebLogic domain namespace in which the ingress will be created + * @param domainUid the WebLogic domainUid which is backend to the ingress + * @param clusterNameMsPortMap the map with key as cluster name and value as managed server port of the cluster + * @return list of ingress hosts or null if got ApiException when calling Kubernetes client API to create ingress + */ + public static List createIngress(String ingressName, + String domainNamespace, + String domainUid, + Map clusterNameMsPortMap) { + return createIngress(ingressName, domainNamespace, domainUid, clusterNameMsPortMap, true); + } + + /** + * Create an ingress for the WebLogic domain with domainUid in the specified domain namespace. + * The ingress host is set to 'domainUid.domainNamespace.clusterName.test'. + * + * @param ingressName name of the ingress to be created + * @param domainNamespace the WebLogic domain namespace in which the ingress will be created + * @param domainUid the WebLogic domainUid which is backend to the ingress + * @param clusterNameMsPortMap the map with key as cluster name and value as managed server port of the cluster + * @param setIngressHost if false will set to any + * @return list of ingress hosts or null if got ApiException when calling Kubernetes client API to create ingress + */ + public static List createIngress(String ingressName, + String domainNamespace, + String domainUid, + Map clusterNameMsPortMap, + boolean setIngressHost) { + + // set the annotation for kubernetes.io/ingress.class to "nginx" + HashMap annotation = new HashMap<>(); + annotation.put("kubernetes.io/ingress.class", INGRESS_NGINX_CLASS); + + List ingressHostList = new ArrayList<>(); + ArrayList ingressRules = new ArrayList<>(); + clusterNameMsPortMap.forEach((clusterName, managedServerPort) -> { + // set the http ingress paths + NetworkingV1beta1HTTPIngressPath httpIngressPath = new NetworkingV1beta1HTTPIngressPath() + .path(null) + .backend(new NetworkingV1beta1IngressBackend() + .serviceName(domainUid + "-cluster-" + clusterName.toLowerCase().replace("_", "-")) + .servicePort(new IntOrString(managedServerPort)) + ); + ArrayList httpIngressPaths = new ArrayList<>(); + httpIngressPaths.add(httpIngressPath); + + // set the ingress rule + String ingressHost = domainUid + "." + domainNamespace + "." + clusterName + ".test"; + if (!setIngressHost) { + ingressHost = ""; + ingressHostList.add("*"); + } + NetworkingV1beta1IngressRule ingressRule = new NetworkingV1beta1IngressRule() + .host(ingressHost) + .http(new NetworkingV1beta1HTTPIngressRuleValue() + .paths(httpIngressPaths)); + + ingressRules.add(ingressRule); + ingressHostList.add(ingressHost); + + }); + + // set the ingress + NetworkingV1beta1Ingress ingress = new NetworkingV1beta1Ingress() + .apiVersion(INGRESS_API_VERSION) + .kind(INGRESS_KIND) + .metadata(new V1ObjectMeta() + .name(ingressName) + .namespace(domainNamespace) + .annotations(annotation)) + .spec(new NetworkingV1beta1IngressSpec() + .rules(ingressRules)); + + // create the ingress + try { + Kubernetes.createIngress(domainNamespace, ingress); + } catch (ApiException apex) { + getLogger().severe("got ApiException while calling createIngress: {0}", apex.getResponseBody()); + return null; + } + return ingressHostList; + } + + /** + * List all of the ingresses in the specified namespace. + * + * @param namespace the namespace to which the ingresses belong + * @return a list of ingress names in the namespace + * @throws ApiException if Kubernetes client API call fails + */ + public static List listIngresses(String namespace) throws ApiException { + + List ingressNames = new ArrayList<>(); + NetworkingV1beta1IngressList ingressList = Kubernetes.listNamespacedIngresses(namespace); + List listOfIngress = ingressList.getItems(); + + listOfIngress.forEach(ingress -> { + if (ingress.getMetadata() != null) { + ingressNames.add(ingress.getMetadata().getName()); + } + }); + + return ingressNames; + } +} diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/ApacheParams.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/ApacheParams.java new file mode 100644 index 00000000000..6bea31a7f16 --- /dev/null +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/ApacheParams.java @@ -0,0 +1,112 @@ +// Copyright (c) 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.weblogic.kubernetes.actions.impl; + +import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * All parameters needed to install Apache ingress controller. + */ +public class ApacheParams { + + // Only add the values which need to be updated here. + // The default values can be found here: + // weblogic-kubernetes-operator/kubernetes/samples/charts/apache-webtier/values.yaml + private String image = null; + private String imagePullSecretsName = null; + private String volumePath = null; + private int httpNodePort = 0; + private int httpsNodePort = 0; + private String virtualHostName = null; + private String customCert = null; + private String customKey = null; + private String domainUID = null; + private HelmParams helmParams; + + public ApacheParams image(String image) { + this.image = image; + return this; + } + + public ApacheParams imagePullSecretsName(String imagePullSecretsName) { + this.imagePullSecretsName = imagePullSecretsName; + return this; + } + + public ApacheParams volumePath(String volumePath) { + this.volumePath = volumePath; + return this; + } + + public ApacheParams httpNodePort(int httpNodePort) { + this.httpNodePort = httpNodePort; + return this; + } + + public ApacheParams httpsNodePort(int httpsNodePort) { + this.httpsNodePort = httpsNodePort; + return this; + } + + public ApacheParams virtualHostName(String virtualHostName) { + this.virtualHostName = virtualHostName; + return this; + } + + public ApacheParams customCert(String customCert) { + this.customCert = customCert; + return this; + } + + public ApacheParams customKey(String customKey) { + this.customKey = customKey; + return this; + } + + public ApacheParams domainUID(String domainUID) { + this.domainUID = domainUID; + return this; + } + + public ApacheParams helmParams(HelmParams helmParams) { + this.helmParams = helmParams; + return this; + } + + public HelmParams getHelmParams() { + return helmParams; + } + + /** + * Loads Helm values into a value map. + * + * @return Map of values + */ + public Map getValues() { + Map values = new HashMap<>(); + + values.put("image", image); + values.put("imagePullSecrets.name", imagePullSecretsName); + values.put("volumePath", volumePath); + + if (httpNodePort >= 0) { + values.put("httpNodePort", httpNodePort); + } + if (httpsNodePort >= 0) { + values.put("httpsNodePort", httpsNodePort); + } + + values.put("virtualHostName", virtualHostName); + values.put("customCert", customCert); + values.put("customKey", customKey); + values.put("domainUID", domainUID); + + values.values().removeIf(Objects::isNull); + return values; + } +} From bb195cb78bc06ceba4e6ef589b480a8fa2d37d34 Mon Sep 17 00:00:00 2001 From: xiancao Date: Mon, 24 Aug 2020 14:43:53 +0000 Subject: [PATCH 02/15] apache lb --- .../weblogic/kubernetes/ItApacheSample.java | 277 ++++++++++++++++++ .../weblogic/kubernetes/TestConstants.java | 7 + .../kubernetes/actions/TestActions.java | 32 ++ .../kubernetes/actions/impl/Apache.java | 10 +- .../kubernetes/actions/impl/ApacheParams.java | 46 ++- .../kubernetes/assertions/TestAssertions.java | 33 +++ .../kubernetes/assertions/impl/Apache.java | 32 ++ .../assertions/impl/Kubernetes.java | 29 ++ .../kubernetes/utils/CommonTestUtils.java | 80 ++++- 9 files changed, 526 insertions(+), 20 deletions(-) create mode 100644 new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItApacheSample.java create mode 100644 new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Apache.java diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItApacheSample.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItApacheSample.java new file mode 100644 index 00000000000..f536e17c4a7 --- /dev/null +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItApacheSample.java @@ -0,0 +1,277 @@ +// Copyright (c) 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.weblogic.kubernetes; + +import java.util.ArrayList; +import java.util.List; + +import io.kubernetes.client.openapi.models.V1EnvVar; +import io.kubernetes.client.openapi.models.V1LocalObjectReference; +import io.kubernetes.client.openapi.models.V1ObjectMeta; +import io.kubernetes.client.openapi.models.V1SecretReference; +import oracle.weblogic.domain.AdminServer; +import oracle.weblogic.domain.AdminService; +import oracle.weblogic.domain.Channel; +import oracle.weblogic.domain.Cluster; +import oracle.weblogic.domain.Configuration; +import oracle.weblogic.domain.Domain; +import oracle.weblogic.domain.DomainSpec; +import oracle.weblogic.domain.Model; +import oracle.weblogic.domain.ServerPod; +import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams; +import oracle.weblogic.kubernetes.annotations.IntegrationTest; +import oracle.weblogic.kubernetes.annotations.Namespaces; +import oracle.weblogic.kubernetes.annotations.tags.MustNotRunInParallel; +import oracle.weblogic.kubernetes.annotations.tags.Slow; +import oracle.weblogic.kubernetes.logging.LoggingFacade; +import org.awaitility.core.ConditionFactory; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.SECONDS; +import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_API_VERSION; +import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST; +import static oracle.weblogic.kubernetes.TestConstants.MANAGED_SERVER_NAME_BASE; +import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_NAME; +import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_TAG; +import static oracle.weblogic.kubernetes.TestConstants.REPO_SECRET_NAME; +//import static oracle.weblogic.kubernetes.actions.TestActions.uninstallApache; +import static oracle.weblogic.kubernetes.assertions.TestAssertions.secretExists; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodExists; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReady; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkServiceExists; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDockerRegistrySecret; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDomainAndVerify; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createSecretWithUsernamePassword; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.dockerLoginAndPushImageToRegistry; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyApache; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyOperator; +import static oracle.weblogic.kubernetes.utils.TestUtils.callWebAppAndCheckForServerNameInResponse; +import static oracle.weblogic.kubernetes.utils.TestUtils.getNextFreePort; +import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; +import static org.awaitility.Awaitility.with; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Verify that Apache and ingress are installed successfully. + */ +@DisplayName("Test Apache is installed successfully") +@IntegrationTest +class ItApacheSample { + + // constants for operator and WebLogic domain + private static String domainUid = "domain1"; + private static String clusterName = "cluster-1"; + private static String adminServerPodName = domainUid + "-admin-server"; + private static String managedServerPrefix = domainUid + "-managed-server"; + + private static int replicaCount = 2; + private static String opNamespace = null; + private static String domainNamespace = null; + private static ConditionFactory withStandardRetryPolicy = null; + + private static HelmParams apacheHelmParams = null; + private static LoggingFacade logger = null; + private static int httpNodePort = 0; + private static int httpsNodePort = 0; + + /** + * Install operator, create a one cluster domain and install Apache. + * + * @param namespaces list of namespaces created by the IntegrationTestWatcher by the + * JUnit engine parameter resolution mechanism + */ + @BeforeAll + public static void init(@Namespaces(2) List namespaces) { + logger = getLogger(); + // create standard, reusable retry/backoff policy + withStandardRetryPolicy = with().pollDelay(2, SECONDS) + .and().with().pollInterval(10, SECONDS) + .atMost(5, MINUTES).await(); + + // get a unique operator namespace + logger.info("Get a unique namespace for operator"); + assertNotNull(namespaces.get(0), "Namespace list is null"); + opNamespace = namespaces.get(0); + + // get a unique domain namespace + logger.info("Get a unique namespace for WebLogic domain"); + assertNotNull(namespaces.get(1), "Namespace list is null"); + domainNamespace = namespaces.get(1); + + // install and verify operator + installAndVerifyOperator(opNamespace, domainNamespace); + + // install and verify Apache + httpNodePort = getNextFreePort(30305, 31305); + httpsNodePort = getNextFreePort(30443, 31443); + apacheHelmParams = + installAndVerifyApache(domainNamespace, httpNodePort, httpsNodePort, domainUid); + + // create and verify one cluster domain + logger.info("Create domain and verify that it's running"); + createAndVerifyDomain(); + + } + + @AfterAll + void tearDown() { + // uninstall Apache + /* + if (apacheHelmParams != null) { + assertThat(uninstallApache(apacheHelmParams)) + .as("Test uninstallApache returns true") + .withFailMessage("uninstallApache() did not return true") + .isTrue(); + } + */ + } + + /** + * The test invokes a webapp to verify the Apache is installed successfully and ready to use. + */ + @Test + @DisplayName("Create the Apache ingress and test Apache load balancing with host name-based routing") + @Slow + @MustNotRunInParallel + public void testApacheDefaultSample() { + + // invoke webapp + String curlRequest = String.format("curl --silent --show-error --noproxy '*' " + + "http://%s:%s/weblogic/sample-war/index.jsp", + K8S_NODEPORT_HOST, httpNodePort); + logger.info("Exec curl command :" + curlRequest); + List managedServers = new ArrayList<>(); + for (int i = 1; i <= replicaCount; i++) { + managedServers.add(MANAGED_SERVER_NAME_BASE + i); + } + assertTrue(callWebAppAndCheckForServerNameInResponse(curlRequest, managedServers,50), + "did not get all servers in the response"); + } + + private static void createAndVerifyDomain() { + // get the pre-built image created by IntegrationTestWatcher + String miiImage = MII_BASIC_IMAGE_NAME + ":" + MII_BASIC_IMAGE_TAG; + + // docker login and push image to docker registry if necessary + dockerLoginAndPushImageToRegistry(miiImage); + + // create docker registry secret to pull the image from registry + if (!secretExists(REPO_SECRET_NAME, domainNamespace)) { + logger.info("Create docker registry secret in namespace {0}", domainNamespace); + assertDoesNotThrow(() -> createDockerRegistrySecret(domainNamespace), + String.format("create Docker Registry Secret failed for %s", REPO_SECRET_NAME)); + } + + // create secret for admin credentials + logger.info("Create secret for admin credentials"); + String adminSecretName = "weblogic-credentials"; + assertDoesNotThrow(() -> createSecretWithUsernamePassword(adminSecretName, domainNamespace, + "weblogic", "welcome1"), + String.format("create secret for admin credentials failed for %s", adminSecretName)); + + // create encryption secret + logger.info("Create encryption secret"); + String encryptionSecretName = "encryptionsecret"; + assertDoesNotThrow(() -> createSecretWithUsernamePassword(encryptionSecretName, domainNamespace, + "weblogicenc", "weblogicenc"), + String.format("create encryption secret failed for %s", encryptionSecretName)); + + // create domain and verify + logger.info("Create model in image domain {0} in namespace {1} using docker image {2}", + domainUid, domainNamespace, miiImage); + createDomainCrAndVerify(adminSecretName, REPO_SECRET_NAME, encryptionSecretName, miiImage); + + // check that admin server pod exists in the domain namespace + logger.info("Checking that admin server pod {0} exists in namespace {1}", + adminServerPodName, domainNamespace); + checkPodExists(adminServerPodName, domainUid, domainNamespace); + + // check that admin server pod is ready + logger.info("Checking that admin server pod {0} is ready in namespace {1}", + adminServerPodName, domainNamespace); + checkPodReady(adminServerPodName, domainUid, domainNamespace); + + // check that admin service exists in the domain namespace + logger.info("Checking that admin service {0} exists in namespace {1}", + adminServerPodName, domainNamespace); + checkServiceExists(adminServerPodName, domainNamespace); + + // check for managed server pods existence in the domain namespace + for (int i = 1; i <= replicaCount; i++) { + String managedServerPodName = managedServerPrefix + i; + + // check that the managed server pod exists + logger.info("Checking that managed server pod {0} exists in namespace {1}", + managedServerPodName, domainNamespace); + checkPodExists(managedServerPodName, domainUid, domainNamespace); + + // check that the managed server pod is ready + logger.info("Checking that managed server pod {0} is ready in namespace {1}", + managedServerPodName, domainNamespace); + checkPodReady(managedServerPodName, domainUid, domainNamespace); + + // check that the managed server service exists in the domain namespace + logger.info("Checking that managed server service {0} exists in namespace {1}", + managedServerPodName, domainNamespace); + checkServiceExists(managedServerPodName, domainNamespace); + } + } + + private static void createDomainCrAndVerify(String adminSecretName, + String repoSecretName, + String encryptionSecretName, + String miiImage) { + // create the domain CR + Domain domain = new Domain() + .apiVersion(DOMAIN_API_VERSION) + .kind("Domain") + .metadata(new V1ObjectMeta() + .name(domainUid) + .namespace(domainNamespace)) + .spec(new DomainSpec() + .domainUid(domainUid) + .domainHomeSourceType("FromModel") + .image(miiImage) + .addImagePullSecretsItem(new V1LocalObjectReference() + .name(repoSecretName)) + .webLogicCredentialsSecret(new V1SecretReference() + .name(adminSecretName) + .namespace(domainNamespace)) + .includeServerOutInPodLog(true) + .serverStartPolicy("IF_NEEDED") + .serverPod(new ServerPod() + .addEnvItem(new V1EnvVar() + .name("JAVA_OPTIONS") + .value("-Dweblogic.StdoutDebugEnabled=false")) + .addEnvItem(new V1EnvVar() + .name("USER_MEM_ARGS") + .value("-Djava.security.egd=file:/dev/./urandom "))) + .adminServer(new AdminServer() + .serverStartState("RUNNING") + .adminService(new AdminService() + .addChannelsItem(new Channel() + .channelName("default") + .nodePort(0)))) + .addClustersItem(new Cluster() + .clusterName(clusterName) + .replicas(replicaCount) + .serverStartState("RUNNING")) + .configuration(new Configuration() + .model(new Model() + .domainType("WLS") + .runtimeEncryptionSecret(encryptionSecretName)) + .introspectorJobActiveDeadlineSeconds(300L))); + + // create domain using model in image + logger.info("Create model in image domain {0} in namespace {1} using docker image {2}", + domainUid, domainNamespace, miiImage); + createDomainAndVerify(domain, domainNamespace); + } +} diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java index 5d2ecc077a2..467f631c015 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java @@ -80,6 +80,9 @@ public interface TestConstants { public static final String STABLE_REPO_NAME = "stable"; public static final String NGINX_CHART_NAME = "nginx-ingress"; + // Apache constants + public static final String APACHE_IMAGE_12213 = "phx.ocir.io/weblogick8s/oracle/apache:12.2.1.3"; + // Traefik constants public static final String TRAEFIK_REPO_URL = "https://containous.github.io/traefik-helm-chart"; public static final String TRAEFIK_REPO_NAME = "traefik"; @@ -93,6 +96,10 @@ public interface TestConstants { public static final String VOYAGER_CHART_NAME = "voyager"; public static final String VOYAGER_CHART_VERSION = "12.0.0"; + // Apache constants + public static final String APACHE_RELEASE_NAME = "apache-release" + BUILD_ID; + public static final String APACHE_SAMPLE_CHART_DIR = "../kubernetes/samples/charts/apache-webtier"; + // ELK Stack and WebLogic logging exporter constants public static final String ELASTICSEARCH_NAME = "elasticsearch"; public static final String ELK_STACK_VERSION = "7.8.1"; diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/TestActions.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/TestActions.java index afc48ca8265..9e86560a1e3 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/TestActions.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/TestActions.java @@ -25,6 +25,8 @@ import io.kubernetes.client.openapi.models.V1ServiceAccount; import io.kubernetes.client.openapi.models.V1ServiceList; import oracle.weblogic.domain.DomainList; +import oracle.weblogic.kubernetes.actions.impl.Apache; +import oracle.weblogic.kubernetes.actions.impl.ApacheParams; import oracle.weblogic.kubernetes.actions.impl.AppBuilder; import oracle.weblogic.kubernetes.actions.impl.AppParams; import oracle.weblogic.kubernetes.actions.impl.ClusterRole; @@ -364,6 +366,16 @@ public static boolean installTraefik(TraefikParams params) { return Traefik.install(params); } + /** + * Install Apache ingress controller. + * + * @param params the parameters to Helm install command, such as release name, namespace, repo url, + * repo name and chart name + * @return true on success, false otherwise + */ + public static boolean installApache(ApacheParams params) { + return Apache.install(params); + } /** * Upgrade NGINX release. @@ -385,6 +397,16 @@ public static boolean upgradeVoyager(VoyagerParams params) { return Voyager.upgrade(params); } + /** + * Upgrade Apache release. + * + * @param params the parameters to Helm upgrade command, such as release name and http/https nodeport + * @return true on success, false otherwise + */ + public static boolean upgradeApache(ApacheParams params) { + return Apache.upgrade(params); + } + /** * Uninstall the NGINX release. * @@ -415,6 +437,16 @@ public static boolean uninstallVoyager(HelmParams params) { return Voyager.uninstall(params); } + /** + * Uninstall the Apache release. + * + * @param params the parameters to Helm uninstall command, such as release name and namespace + * @return true on success, false otherwise + */ + public static boolean uninstallApache(HelmParams params) { + return Apache.uninstall(params); + } + /** * Create an ingress for the WebLogic domain with domainUid in the specified domain namespace. * The ingress host is set to 'domainUid.clusterName.test'. diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/Apache.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/Apache.java index 575bbe254e2..ab73af5337e 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/Apache.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/Apache.java @@ -3,6 +3,11 @@ package oracle.weblogic.kubernetes.actions.impl; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import io.kubernetes.client.custom.IntOrString; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.models.NetworkingV1beta1HTTPIngressPath; @@ -17,11 +22,6 @@ import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams; import oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; /** diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/ApacheParams.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/ApacheParams.java index 6bea31a7f16..155055775ea 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/ApacheParams.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/ApacheParams.java @@ -3,12 +3,12 @@ package oracle.weblogic.kubernetes.actions.impl; -import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams; - import java.util.HashMap; import java.util.Map; import java.util.Objects; +import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams; + /** * All parameters needed to install Apache ingress controller. */ @@ -17,8 +17,20 @@ public class ApacheParams { // Only add the values which need to be updated here. // The default values can be found here: // weblogic-kubernetes-operator/kubernetes/samples/charts/apache-webtier/values.yaml + private static final String IMAGE = "image"; + private static final String IMAGE_PULL_POLICY = "imagePullPolicy"; + private static final String IMAGE_PULL_SECRETS = "imagePullSecrets"; + private static final String VOLUME_PATH = "volumePath"; + private static final String HTTP_NODEPORT = "httpNodePort"; + private static final String HTTPS_NODEPORT = "httpsNodePort"; + private static final String VIRTUAL_HOSTNAME = "virtualHostName"; + private static final String CUSTOM_CERT = "customCert"; + private static final String CUSTOM_KEY = "customKey"; + private static final String DOMAIN_UID = "domainUID"; + private String image = null; - private String imagePullSecretsName = null; + private String imagePullPolicy = null; + private Map imagePullSecrets = null; private String volumePath = null; private int httpNodePort = 0; private int httpsNodePort = 0; @@ -33,8 +45,13 @@ public ApacheParams image(String image) { return this; } - public ApacheParams imagePullSecretsName(String imagePullSecretsName) { - this.imagePullSecretsName = imagePullSecretsName; + public ApacheParams imagePullPolicy(String imagePullPolicy) { + this.imagePullPolicy = imagePullPolicy; + return this; + } + + public ApacheParams imagePullSecrets(Map imagePullSecrets) { + this.imagePullSecrets = imagePullSecrets; return this; } @@ -90,21 +107,22 @@ public HelmParams getHelmParams() { public Map getValues() { Map values = new HashMap<>(); - values.put("image", image); - values.put("imagePullSecrets.name", imagePullSecretsName); - values.put("volumePath", volumePath); + values.put(IMAGE, image); + values.put(IMAGE_PULL_POLICY, imagePullPolicy); + values.put(IMAGE_PULL_SECRETS, imagePullSecrets); + values.put(VOLUME_PATH, volumePath); if (httpNodePort >= 0) { - values.put("httpNodePort", httpNodePort); + values.put(HTTP_NODEPORT, httpNodePort); } if (httpsNodePort >= 0) { - values.put("httpsNodePort", httpsNodePort); + values.put(HTTPS_NODEPORT, httpsNodePort); } - values.put("virtualHostName", virtualHostName); - values.put("customCert", customCert); - values.put("customKey", customKey); - values.put("domainUID", domainUID); + values.put(VIRTUAL_HOSTNAME, virtualHostName); + values.put(CUSTOM_CERT, customCert); + values.put(CUSTOM_KEY, customKey); + values.put(DOMAIN_UID, domainUID); values.values().removeIf(Objects::isNull); return values; diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java index cbb4c8a1e6c..452a7d7f733 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java @@ -8,7 +8,10 @@ import java.util.concurrent.Callable; import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.models.V1Secret; import oracle.weblogic.kubernetes.actions.impl.LoggingExporter; +import oracle.weblogic.kubernetes.actions.impl.Secret; +import oracle.weblogic.kubernetes.assertions.impl.Apache; import oracle.weblogic.kubernetes.assertions.impl.Application; import oracle.weblogic.kubernetes.assertions.impl.ClusterRole; import oracle.weblogic.kubernetes.assertions.impl.ClusterRoleBinding; @@ -65,6 +68,16 @@ public static Callable isNginxReady(String namespace) { return Nginx.isReady(namespace); } + /** + * Check if there are ready Apache pods in the specified namespace. + * + * @param namespace in which to check if APache pods are in the ready state + * @return true if there are ready Apache pods in the specified namespace , false otherwise + */ + public static Callable isApacheReady(String namespace) { + return Apache.isReady(namespace); + } + /** * Check traefik controller pod is ready in the specified namespace. * @@ -597,4 +610,24 @@ public static boolean clusterRoleExists(String clusterRoleName) throws ApiExcept public static boolean clusterRoleBindingExists(String clusterRoleBindingName) throws ApiException { return ClusterRoleBinding.clusterRoleBindingExists(clusterRoleBindingName); } + + /** + * Check whether the secret exists in the specified namespace. + * + * @param secretName name of the secret + * @param namespace namespace in which the secret exists + * @return true if secret exists, false otherwise + */ + public static boolean secretExists(String secretName, String namespace) { + for (V1Secret secret : Secret.listSecrets(namespace).getItems()) { + if (secret.getMetadata() != null) { + String name = secret.getMetadata().getName(); + if (name != null && name.equals(secretName)) { + return true; + } + } + } + + return false; + } } diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Apache.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Apache.java new file mode 100644 index 00000000000..ab2b5522964 --- /dev/null +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Apache.java @@ -0,0 +1,32 @@ +// Copyright (c) 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.weblogic.kubernetes.assertions.impl; + +import java.util.concurrent.Callable; + +/** + * Assertions for Apache ingress controller. + */ +public class Apache { + + /** + * Check if the Apache pod is running in the specified namespace. + * + * @param namespace in which to check if the Apache pod is running + * @return true if the Apache pod is running, false otherwise + */ + public static Callable isRunning(String namespace) { + return () -> Kubernetes.isApachePodRunning(namespace); + } + + /** + * Check if the Apache pod is ready in the specified namespace. + * + * @param namespace in which to check the Apache pod is ready + * @return true if the Apache pod is in the ready state, false otherwise + */ + public static Callable isReady(String namespace) { + return () -> Kubernetes.isApachePodReady(namespace); + } +} diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Kubernetes.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Kubernetes.java index 5c12999eb40..b15c27caa18 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Kubernetes.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Kubernetes.java @@ -38,6 +38,7 @@ import org.joda.time.DateTime; import static io.kubernetes.client.util.Yaml.dump; +import static oracle.weblogic.kubernetes.TestConstants.APACHE_RELEASE_NAME; import static oracle.weblogic.kubernetes.TestConstants.TRAEFIK_RELEASE_NAME; import static oracle.weblogic.kubernetes.actions.TestActions.getPodRestartVersion; import static oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes.getPodCreationTimestamp; @@ -344,6 +345,34 @@ public static boolean isNginxPodReady(String namespace) throws ApiException { return isPodReady(namespace, labelSelector, "nginx-ingress-controller"); } + /** + * Checks if a Apache pod is running in the specified namespace. + * The method assumes that the Apache pod name contains "APACHE_RELEAE_NAME-apache-webtier". + * + * @param namespace in which to check if the Apache pod is running + * @return true if the pod is running, otherwise false + * @throws ApiException if Kubernetes client API call fails + */ + public static boolean isApachePodRunning(String namespace) throws ApiException { + + return isPodRunning(namespace, null, + APACHE_RELEASE_NAME + "-" + namespace.substring(3) + "-apache-webtier"); + } + + /** + * Check whether the Apache pod is ready in the specified namespace. + * The method assumes that the Apache pod name starts with "APACHE_RELEASE_NAME_apache-webtier". + * + * @param namespace in which to check if the Apache pod is ready + * @return true if the pod is in the ready state, false otherwise + * @throws ApiException if Kubernetes client API call fails + */ + public static boolean isApachePodReady(String namespace) throws ApiException { + String labelSelector = null; + return isPodReady(namespace, labelSelector, + APACHE_RELEASE_NAME + "-" + namespace.substring(3) + "-apache-webtier"); + } + /** * Checks if traefik pod is running in the specified namespace. * diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java index 317207b443f..e8aa3fa71cb 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java @@ -52,6 +52,7 @@ import oracle.weblogic.domain.Domain; import oracle.weblogic.kubernetes.TestConstants; import oracle.weblogic.kubernetes.actions.TestActions; +import oracle.weblogic.kubernetes.actions.impl.ApacheParams; import oracle.weblogic.kubernetes.actions.impl.GrafanaParams; import oracle.weblogic.kubernetes.actions.impl.LoggingExporterParams; import oracle.weblogic.kubernetes.actions.impl.NginxParams; @@ -71,6 +72,9 @@ import static java.nio.file.Files.readString; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; +import static oracle.weblogic.kubernetes.TestConstants.APACHE_IMAGE_12213; +import static oracle.weblogic.kubernetes.TestConstants.APACHE_RELEASE_NAME; +import static oracle.weblogic.kubernetes.TestConstants.APACHE_SAMPLE_CHART_DIR; import static oracle.weblogic.kubernetes.TestConstants.APPSCODE_REPO_NAME; import static oracle.weblogic.kubernetes.TestConstants.APPSCODE_REPO_URL; import static oracle.weblogic.kubernetes.TestConstants.DEFAULT_EXTERNAL_REST_IDENTITY_SECRET_NAME; @@ -145,6 +149,7 @@ import static oracle.weblogic.kubernetes.actions.TestActions.getPodCreationTimestamp; import static oracle.weblogic.kubernetes.actions.TestActions.getPodLog; import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort; +import static oracle.weblogic.kubernetes.actions.TestActions.installApache; import static oracle.weblogic.kubernetes.actions.TestActions.installElasticsearch; import static oracle.weblogic.kubernetes.actions.TestActions.installGrafana; import static oracle.weblogic.kubernetes.actions.TestActions.installKibana; @@ -166,6 +171,7 @@ import static oracle.weblogic.kubernetes.assertions.TestAssertions.credentialsValid; import static oracle.weblogic.kubernetes.assertions.TestAssertions.doesImageExist; import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainExists; +import static oracle.weblogic.kubernetes.assertions.TestAssertions.isApacheReady; import static oracle.weblogic.kubernetes.assertions.TestAssertions.isElkStackPodReady; import static oracle.weblogic.kubernetes.assertions.TestAssertions.isGrafanaReady; import static oracle.weblogic.kubernetes.assertions.TestAssertions.isHelmReleaseDeployed; @@ -183,6 +189,7 @@ import static oracle.weblogic.kubernetes.assertions.TestAssertions.podStateNotChanged; import static oracle.weblogic.kubernetes.assertions.TestAssertions.pvExists; import static oracle.weblogic.kubernetes.assertions.TestAssertions.pvcExists; +import static oracle.weblogic.kubernetes.assertions.TestAssertions.secretExists; import static oracle.weblogic.kubernetes.assertions.TestAssertions.serviceDoesNotExist; import static oracle.weblogic.kubernetes.assertions.TestAssertions.serviceExists; import static oracle.weblogic.kubernetes.utils.ExecCommand.exec; @@ -605,6 +612,78 @@ public static HelmParams installAndVerifyVoyager(String voyagerNamespace, return voyagerHelmParams; } + /** + * Install Apache and wait up to five minutes until the Apache pod is ready. + * + * @param apacheNamespace the namespace in which the Apache will be installed + * @param httpNodePort the http nodeport of Apache + * @param httpsNodePort the https nodeport of Apache + * @param domainUid the uid of the domain to which Apache will route the services + * @return the Apache Helm installation parameters + */ + public static HelmParams installAndVerifyApache(String apacheNamespace, + int httpNodePort, + int httpsNodePort, + String domainUid) { + LoggingFacade logger = getLogger(); + + // Create Docker registry secret in the operator namespace to pull the image from repository + if (!secretExists(REPO_SECRET_NAME, apacheNamespace)) { + logger.info("Creating Docker registry secret in namespace {0}", apacheNamespace); + createDockerRegistrySecret(apacheNamespace); + } + + // map with secret + Map secretNameMap = new HashMap<>(); + secretNameMap.put("name", REPO_SECRET_NAME); + + // Helm install parameters + HelmParams apacheHelmParams = new HelmParams() + .releaseName(APACHE_RELEASE_NAME + "-" + apacheNamespace.substring(3)) + .namespace(apacheNamespace) + .chartDir(APACHE_SAMPLE_CHART_DIR); + + // Apache chart values to override + ApacheParams apacheParams = new ApacheParams() + .helmParams(apacheHelmParams) + .imagePullSecrets(secretNameMap) + .image(APACHE_IMAGE_12213) + .domainUID(domainUid); + + if (httpNodePort >= 0 && httpsNodePort >= 0) { + apacheParams + .httpNodePort(httpNodePort) + .httpsNodePort(httpsNodePort); + } + + // install Apache + assertThat(installApache(apacheParams)) + .as("Test Apache installation succeeds") + .withFailMessage("Apache installation is failed") + .isTrue(); + + // verify that Apache is installed + logger.info("Checking Apache release {0} status in namespace {1}", + APACHE_RELEASE_NAME, apacheNamespace); + assertTrue(isHelmReleaseDeployed(APACHE_RELEASE_NAME, apacheNamespace), + String.format("Apache release %s is not in deployed status in namespace %s", + APACHE_RELEASE_NAME, apacheNamespace)); + logger.info("Apache release {0} status is deployed in namespace {1}", + APACHE_RELEASE_NAME, apacheNamespace); + + // wait until the Apache pod is ready. + withStandardRetryPolicy + .conditionEvaluationListener( + condition -> logger.info( + "Waiting for Apache to be ready in namespace {0} (elapsed time {1}ms, remaining time {2}ms)", + apacheNamespace, + condition.getElapsedTimeInMS(), + condition.getRemainingTimeInMS())) + .until(assertDoesNotThrow(() -> isApacheReady(apacheNamespace), "isApacheReady failed with ApiException")); + + return apacheHelmParams; + } + /** * Uninstall Elasticsearch. * @@ -2581,5 +2660,4 @@ public static void verifyDefaultTokenExists() { return false; }); } - } From 6596ebe80e69d6abc098d278efa11279682cb631 Mon Sep 17 00:00:00 2001 From: xiancao Date: Wed, 26 Aug 2020 15:29:43 +0000 Subject: [PATCH 03/15] add apache tests to ItTwoDomainsLoadBalancers --- .../weblogic/kubernetes/ItApacheSample.java | 277 ------------------ .../kubernetes/ItTwoDomainsLoadBalancers.java | 199 +++++++++---- .../kubernetes/utils/CommonTestUtils.java | 99 ++++++- 3 files changed, 232 insertions(+), 343 deletions(-) delete mode 100644 new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItApacheSample.java diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItApacheSample.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItApacheSample.java deleted file mode 100644 index f536e17c4a7..00000000000 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItApacheSample.java +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright (c) 2020, Oracle Corporation and/or its affiliates. -// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. - -package oracle.weblogic.kubernetes; - -import java.util.ArrayList; -import java.util.List; - -import io.kubernetes.client.openapi.models.V1EnvVar; -import io.kubernetes.client.openapi.models.V1LocalObjectReference; -import io.kubernetes.client.openapi.models.V1ObjectMeta; -import io.kubernetes.client.openapi.models.V1SecretReference; -import oracle.weblogic.domain.AdminServer; -import oracle.weblogic.domain.AdminService; -import oracle.weblogic.domain.Channel; -import oracle.weblogic.domain.Cluster; -import oracle.weblogic.domain.Configuration; -import oracle.weblogic.domain.Domain; -import oracle.weblogic.domain.DomainSpec; -import oracle.weblogic.domain.Model; -import oracle.weblogic.domain.ServerPod; -import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams; -import oracle.weblogic.kubernetes.annotations.IntegrationTest; -import oracle.weblogic.kubernetes.annotations.Namespaces; -import oracle.weblogic.kubernetes.annotations.tags.MustNotRunInParallel; -import oracle.weblogic.kubernetes.annotations.tags.Slow; -import oracle.weblogic.kubernetes.logging.LoggingFacade; -import org.awaitility.core.ConditionFactory; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -import static java.util.concurrent.TimeUnit.MINUTES; -import static java.util.concurrent.TimeUnit.SECONDS; -import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_API_VERSION; -import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST; -import static oracle.weblogic.kubernetes.TestConstants.MANAGED_SERVER_NAME_BASE; -import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_NAME; -import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_TAG; -import static oracle.weblogic.kubernetes.TestConstants.REPO_SECRET_NAME; -//import static oracle.weblogic.kubernetes.actions.TestActions.uninstallApache; -import static oracle.weblogic.kubernetes.assertions.TestAssertions.secretExists; -import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodExists; -import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReady; -import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkServiceExists; -import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDockerRegistrySecret; -import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDomainAndVerify; -import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createSecretWithUsernamePassword; -import static oracle.weblogic.kubernetes.utils.CommonTestUtils.dockerLoginAndPushImageToRegistry; -import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyApache; -import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyOperator; -import static oracle.weblogic.kubernetes.utils.TestUtils.callWebAppAndCheckForServerNameInResponse; -import static oracle.weblogic.kubernetes.utils.TestUtils.getNextFreePort; -import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; -import static org.awaitility.Awaitility.with; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Verify that Apache and ingress are installed successfully. - */ -@DisplayName("Test Apache is installed successfully") -@IntegrationTest -class ItApacheSample { - - // constants for operator and WebLogic domain - private static String domainUid = "domain1"; - private static String clusterName = "cluster-1"; - private static String adminServerPodName = domainUid + "-admin-server"; - private static String managedServerPrefix = domainUid + "-managed-server"; - - private static int replicaCount = 2; - private static String opNamespace = null; - private static String domainNamespace = null; - private static ConditionFactory withStandardRetryPolicy = null; - - private static HelmParams apacheHelmParams = null; - private static LoggingFacade logger = null; - private static int httpNodePort = 0; - private static int httpsNodePort = 0; - - /** - * Install operator, create a one cluster domain and install Apache. - * - * @param namespaces list of namespaces created by the IntegrationTestWatcher by the - * JUnit engine parameter resolution mechanism - */ - @BeforeAll - public static void init(@Namespaces(2) List namespaces) { - logger = getLogger(); - // create standard, reusable retry/backoff policy - withStandardRetryPolicy = with().pollDelay(2, SECONDS) - .and().with().pollInterval(10, SECONDS) - .atMost(5, MINUTES).await(); - - // get a unique operator namespace - logger.info("Get a unique namespace for operator"); - assertNotNull(namespaces.get(0), "Namespace list is null"); - opNamespace = namespaces.get(0); - - // get a unique domain namespace - logger.info("Get a unique namespace for WebLogic domain"); - assertNotNull(namespaces.get(1), "Namespace list is null"); - domainNamespace = namespaces.get(1); - - // install and verify operator - installAndVerifyOperator(opNamespace, domainNamespace); - - // install and verify Apache - httpNodePort = getNextFreePort(30305, 31305); - httpsNodePort = getNextFreePort(30443, 31443); - apacheHelmParams = - installAndVerifyApache(domainNamespace, httpNodePort, httpsNodePort, domainUid); - - // create and verify one cluster domain - logger.info("Create domain and verify that it's running"); - createAndVerifyDomain(); - - } - - @AfterAll - void tearDown() { - // uninstall Apache - /* - if (apacheHelmParams != null) { - assertThat(uninstallApache(apacheHelmParams)) - .as("Test uninstallApache returns true") - .withFailMessage("uninstallApache() did not return true") - .isTrue(); - } - */ - } - - /** - * The test invokes a webapp to verify the Apache is installed successfully and ready to use. - */ - @Test - @DisplayName("Create the Apache ingress and test Apache load balancing with host name-based routing") - @Slow - @MustNotRunInParallel - public void testApacheDefaultSample() { - - // invoke webapp - String curlRequest = String.format("curl --silent --show-error --noproxy '*' " - + "http://%s:%s/weblogic/sample-war/index.jsp", - K8S_NODEPORT_HOST, httpNodePort); - logger.info("Exec curl command :" + curlRequest); - List managedServers = new ArrayList<>(); - for (int i = 1; i <= replicaCount; i++) { - managedServers.add(MANAGED_SERVER_NAME_BASE + i); - } - assertTrue(callWebAppAndCheckForServerNameInResponse(curlRequest, managedServers,50), - "did not get all servers in the response"); - } - - private static void createAndVerifyDomain() { - // get the pre-built image created by IntegrationTestWatcher - String miiImage = MII_BASIC_IMAGE_NAME + ":" + MII_BASIC_IMAGE_TAG; - - // docker login and push image to docker registry if necessary - dockerLoginAndPushImageToRegistry(miiImage); - - // create docker registry secret to pull the image from registry - if (!secretExists(REPO_SECRET_NAME, domainNamespace)) { - logger.info("Create docker registry secret in namespace {0}", domainNamespace); - assertDoesNotThrow(() -> createDockerRegistrySecret(domainNamespace), - String.format("create Docker Registry Secret failed for %s", REPO_SECRET_NAME)); - } - - // create secret for admin credentials - logger.info("Create secret for admin credentials"); - String adminSecretName = "weblogic-credentials"; - assertDoesNotThrow(() -> createSecretWithUsernamePassword(adminSecretName, domainNamespace, - "weblogic", "welcome1"), - String.format("create secret for admin credentials failed for %s", adminSecretName)); - - // create encryption secret - logger.info("Create encryption secret"); - String encryptionSecretName = "encryptionsecret"; - assertDoesNotThrow(() -> createSecretWithUsernamePassword(encryptionSecretName, domainNamespace, - "weblogicenc", "weblogicenc"), - String.format("create encryption secret failed for %s", encryptionSecretName)); - - // create domain and verify - logger.info("Create model in image domain {0} in namespace {1} using docker image {2}", - domainUid, domainNamespace, miiImage); - createDomainCrAndVerify(adminSecretName, REPO_SECRET_NAME, encryptionSecretName, miiImage); - - // check that admin server pod exists in the domain namespace - logger.info("Checking that admin server pod {0} exists in namespace {1}", - adminServerPodName, domainNamespace); - checkPodExists(adminServerPodName, domainUid, domainNamespace); - - // check that admin server pod is ready - logger.info("Checking that admin server pod {0} is ready in namespace {1}", - adminServerPodName, domainNamespace); - checkPodReady(adminServerPodName, domainUid, domainNamespace); - - // check that admin service exists in the domain namespace - logger.info("Checking that admin service {0} exists in namespace {1}", - adminServerPodName, domainNamespace); - checkServiceExists(adminServerPodName, domainNamespace); - - // check for managed server pods existence in the domain namespace - for (int i = 1; i <= replicaCount; i++) { - String managedServerPodName = managedServerPrefix + i; - - // check that the managed server pod exists - logger.info("Checking that managed server pod {0} exists in namespace {1}", - managedServerPodName, domainNamespace); - checkPodExists(managedServerPodName, domainUid, domainNamespace); - - // check that the managed server pod is ready - logger.info("Checking that managed server pod {0} is ready in namespace {1}", - managedServerPodName, domainNamespace); - checkPodReady(managedServerPodName, domainUid, domainNamespace); - - // check that the managed server service exists in the domain namespace - logger.info("Checking that managed server service {0} exists in namespace {1}", - managedServerPodName, domainNamespace); - checkServiceExists(managedServerPodName, domainNamespace); - } - } - - private static void createDomainCrAndVerify(String adminSecretName, - String repoSecretName, - String encryptionSecretName, - String miiImage) { - // create the domain CR - Domain domain = new Domain() - .apiVersion(DOMAIN_API_VERSION) - .kind("Domain") - .metadata(new V1ObjectMeta() - .name(domainUid) - .namespace(domainNamespace)) - .spec(new DomainSpec() - .domainUid(domainUid) - .domainHomeSourceType("FromModel") - .image(miiImage) - .addImagePullSecretsItem(new V1LocalObjectReference() - .name(repoSecretName)) - .webLogicCredentialsSecret(new V1SecretReference() - .name(adminSecretName) - .namespace(domainNamespace)) - .includeServerOutInPodLog(true) - .serverStartPolicy("IF_NEEDED") - .serverPod(new ServerPod() - .addEnvItem(new V1EnvVar() - .name("JAVA_OPTIONS") - .value("-Dweblogic.StdoutDebugEnabled=false")) - .addEnvItem(new V1EnvVar() - .name("USER_MEM_ARGS") - .value("-Djava.security.egd=file:/dev/./urandom "))) - .adminServer(new AdminServer() - .serverStartState("RUNNING") - .adminService(new AdminService() - .addChannelsItem(new Channel() - .channelName("default") - .nodePort(0)))) - .addClustersItem(new Cluster() - .clusterName(clusterName) - .replicas(replicaCount) - .serverStartState("RUNNING")) - .configuration(new Configuration() - .model(new Model() - .domainType("WLS") - .runtimeEncryptionSecret(encryptionSecretName)) - .introspectorJobActiveDeadlineSeconds(300L))); - - // create domain using model in image - logger.info("Create model in image domain {0} in namespace {1} using docker image {2}", - domainUid, domainNamespace, miiImage); - createDomainAndVerify(domain, domainNamespace); - } -} diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java index 962334961ba..04a383cb450 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java @@ -14,6 +14,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -82,6 +83,8 @@ import static oracle.weblogic.kubernetes.TestConstants.ADMIN_PASSWORD_DEFAULT; import static oracle.weblogic.kubernetes.TestConstants.ADMIN_SERVER_NAME_BASE; import static oracle.weblogic.kubernetes.TestConstants.ADMIN_USERNAME_DEFAULT; +import static oracle.weblogic.kubernetes.TestConstants.APACHE_IMAGE_12213; +import static oracle.weblogic.kubernetes.TestConstants.APACHE_RELEASE_NAME; import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_API_VERSION; import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_VERSION; import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST; @@ -111,6 +114,7 @@ import static oracle.weblogic.kubernetes.actions.TestActions.listSecrets; import static oracle.weblogic.kubernetes.actions.TestActions.shutdownDomain; import static oracle.weblogic.kubernetes.actions.TestActions.startDomain; +import static oracle.weblogic.kubernetes.actions.TestActions.uninstallApache; import static oracle.weblogic.kubernetes.actions.TestActions.uninstallOperator; import static oracle.weblogic.kubernetes.actions.TestActions.uninstallTraefik; import static oracle.weblogic.kubernetes.actions.TestActions.uninstallVoyager; @@ -127,6 +131,7 @@ import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createSecretWithTLSCertKey; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createSecretWithUsernamePassword; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getPodCreationTime; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyApache; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyOperator; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyTraefik; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyVoyager; @@ -175,16 +180,13 @@ public class ItTwoDomainsLoadBalancers { private static Path clusterViewAppPath; private static LoggingFacade logger = null; private static Path dstFile = null; - - // create standard, reusable retry/backoff policy - private static final ConditionFactory withStandardRetryPolicy - = with().pollDelay(2, SECONDS) - .and().with().pollInterval(10, SECONDS) - .atMost(5, MINUTES).await(); + private static HelmParams apacheHelmParams1 = null; + private static HelmParams apacheHelmParams2 = null; // domain constants private final String clusterName = "cluster-1"; private final int replicaCount = 2; + private final String managedServerPort = "8001"; private int t3ChannelPort = 0; private int replicasAfterScale; @@ -274,8 +276,8 @@ public void testTwoDomainsManagedByTwoOperators() { // restart domain1 and verify no impact on domain2 restartDomain1AndVerifyNoImpactOnDomain2(replicasAfterScale, domain1Namespace, domain2Namespace); - // shutdown both domains and verify the pods were shutdown - shutdownBothDomainsAndVerify(domainNamespaces); + // shutdown domain2 and verify the pods were shutdown + shutdownDomainAndVerify(domain2Namespace, domain2Uid); } /** @@ -332,13 +334,16 @@ public void testDeployAppAndInstallIngressControllers() { "Application archive is not available"); clusterViewAppPath = Paths.get(distDir.toString(), "clusterview.war"); - // deploy clusterview application + // deploy clusterview application in default namespace for (String domainUid : domainUids) { // admin/managed server name here should match with model yaml in MII_BASIC_WDT_MODEL_FILE String adminServerPodName = domainUid + "-admin-server"; - deployApplication(domainUid, adminServerPodName); + deployApplication(defaultNamespace, domainUid, adminServerPodName); } + // deploy clusterview application in domain1Namespace + deployApplication(domain1Namespace, domain1Uid, domain1Uid + "-admin-server"); + // create TLS secret for https traffic for (String domainUid : domainUids) { createCertKeyFiles(domainUid); @@ -348,6 +353,18 @@ public void testDeployAppAndInstallIngressControllers() { // create loadbalancing rules for Traefik createTraefikIngressRoutingRules(); createVoyagerIngressRoutingRules(); + + // install and verify Apache + apacheHelmParams1 = assertDoesNotThrow( + () -> installAndVerifyApache(domain1Namespace, APACHE_IMAGE_12213, 0, 0, domain1Uid)); + + LinkedHashMap clusterNamePortMap = new LinkedHashMap<>(); + for (int i = 0; i < numberOfDomains; i++) { + clusterNamePortMap.put(domainUids.get(i) + "-cluster-cluster-1", managedServerPort); + } + apacheHelmParams2 = assertDoesNotThrow( + () -> installAndVerifyApache(defaultNamespace, APACHE_IMAGE_12213, 0, 0, domain1Uid, + RESULTS_ROOT, "apache-sample-host", clusterNamePortMap)); } /** @@ -378,7 +395,8 @@ public void testTraefikHttpHostRoutingAcrossDomains() { // verify load balancing works when 2 domains are running in the same namespace logger.info("Verifying http traffic"); for (String domainUid : domainUids) { - verifyClusterLoadbalancing(domainUid, "http", getTraefikLbNodePort(false)); + verifyClusterLoadbalancing(domainUid, defaultNamespace,"http", getTraefikLbNodePort(false), + replicaCount, true, ""); } } @@ -394,7 +412,8 @@ public void testTraefikHostHttpsRoutingAcrossDomains() { logger.info("Verifying https traffic"); for (String domainUid : domainUids) { - verifyClusterLoadbalancing(domainUid, "https", getTraefikLbNodePort(true)); + verifyClusterLoadbalancing(domainUid, defaultNamespace, "https", getTraefikLbNodePort(true), + replicaCount, true, ""); } } @@ -412,7 +431,44 @@ public void testVoyagerHostHttpRoutingAcrossDomains() { logger.info("Verifying http traffic"); for (String domainUid : domainUids) { String ingressName = domainUid + "-ingress-host-routing"; - verifyClusterLoadbalancing(domainUid, "http", getVoyagerLbNodePort(ingressName)); + verifyClusterLoadbalancing(domainUid, defaultNamespace, "http", getVoyagerLbNodePort(ingressName), + replicaCount, true, ""); + } + } + + /** + * Verify Apache load balancer default sample. + */ + @Order(8) + @Test + @DisplayName("verify Apache loadbalancer default sample through HTTP channel") + public void testApacheDefaultSample() { + + // verify Apache default sample + logger.info("Verifying Apache default sample"); + int httpNodePort = getApacheNodePort(domain1Namespace, "http"); + verifyClusterLoadbalancing(domain1Uid, domain1Namespace, "http", httpNodePort, 3, + false, "weblogic"); + } + + /** + * Verify Apache load balancer custom sample. + */ + @Order(9) + @Test + @DisplayName("verify Apache loadbalancer custom sample through HTTP and HTTPS channel") + public void testApacheCustomSample() { + + // verify Apache custom sample + logger.info("Verifying Apache custom sample"); + for (int i = 1; i <= numberOfDomains; i++) { + int httpNodePort = getApacheNodePort(defaultNamespace, "http"); + verifyClusterLoadbalancing(domainUids.get(i - 1), defaultNamespace, "http", httpNodePort, replicaCount, + false, "weblogic" + i); + + int httpsNodePort = getApacheNodePort(defaultNamespace, "https"); + verifyClusterLoadbalancing(domainUids.get(i - 1), defaultNamespace, "https", httpsNodePort, replicaCount, + false, "weblogic" + i); } } @@ -437,6 +493,21 @@ public void tearDownAll() { .isTrue(); } + // uninstall Apache + if (apacheHelmParams1 != null) { + assertThat(uninstallApache(apacheHelmParams1)) + .as("Test uninstallApache returns true") + .withFailMessage("uninstallApache() did not return true") + .isTrue(); + } + + if (apacheHelmParams2 != null) { + assertThat(uninstallApache(apacheHelmParams2)) + .as("Test uninstallApache returns true") + .withFailMessage("uninstallApache() did not return true") + .isTrue(); + } + // uninstall operator which manages default namespace logger.info("uninstalling operator which manages default namespace"); if (operatorHelmParams != null) { @@ -912,38 +983,27 @@ private void getBothDomainsPodsOriginalCreationTimestamp(List domainUids } /** - * Shutdown both domains and verify all the server pods were shutdown. + * Shutdown domain and verify all the server pods were shutdown. * - * @param domainNamespaces list of domain namespaces + * @param domainNamespace the namespace where the domain exists + * @param domainUid the uid of the domain */ - private void shutdownBothDomainsAndVerify(List domainNamespaces) { + private void shutdownDomainAndVerify(String domainNamespace, String domainUid) { - // shutdown both domains - logger.info("Shutting down both domains"); - for (int i = 0; i < numberOfDomains; i++) { - shutdownDomain(domainUids.get(i), domainNamespaces.get(i)); - } + // shutdown domain + logger.info("Shutting down domain {0} in namespace {1}", domainUid, domainNamespace); + shutdownDomain(domainUid, domainNamespace); // verify all the pods were shutdown - logger.info("Verifying all server pods were shutdown for both domains"); - for (int i = 0; i < numberOfDomains; i++) { - // check admin server pod was shutdown - checkPodDoesNotExist(domainUids.get(i) + "-" + ADMIN_SERVER_NAME_BASE, - domainUids.get(i), domainNamespaces.get(i)); - - for (int j = 1; j <= replicaCount; j++) { - String managedServerPodName = domainUids.get(i) + "-" + MANAGED_SERVER_NAME_BASE + j; - checkPodDoesNotExist(managedServerPodName, domainUids.get(i), domainNamespaces.get(i)); - } - } - - // check the scaled up managed servers in domain1 were shutdown - logger.info("Verifying the scaled up managed servers in domain1 were shutdown"); - for (int i = replicaCount + 1; i <= replicasAfterScale; i++) { - String managedServerPodName = domain1Uid + "-" + MANAGED_SERVER_NAME_BASE + i; - checkPodDoesNotExist(managedServerPodName, domain1Uid, domain1Namespace); + logger.info("Verifying all server pods were shutdown for the domain"); + // check admin server pod was shutdown + checkPodDoesNotExist(domainUid + "-" + ADMIN_SERVER_NAME_BASE, + domainUid, domainNamespace); + + for (int j = 1; j <= replicaCount; j++) { + String managedServerPodName = domainUid + "-" + MANAGED_SERVER_NAME_BASE + j; + checkPodDoesNotExist(managedServerPodName, domainUid, domainNamespace); } - } /** @@ -1233,14 +1293,25 @@ private static int getVoyagerLbNodePort(String ingressName) { return ingressServiceNodePort; } - private static void deployApplication(String domainUid, String adminServerPodName) { + private static int getApacheNodePort(String namespace, String channelName) { + String apacheServiceName = APACHE_RELEASE_NAME + "-" + namespace.substring(3) + "-apache-webtier"; + + // get ingress service Nodeport + int apacheHttpNodePort = assertDoesNotThrow(() -> + getServiceNodePort(namespace, apacheServiceName, channelName), + "Getting Apache load balancer service node port failed"); + logger.info("Node port for {0} is: {1} :", apacheServiceName, apacheHttpNodePort); + return apacheHttpNodePort; + } + + private static void deployApplication(String namespace, String domainUid, String adminServerPodName) { logger.info("Getting node port for admin server default channel"); int serviceNodePort = assertDoesNotThrow(() -> - getServiceNodePort(defaultNamespace, adminServerPodName + "-external", "default"), + getServiceNodePort(namespace, adminServerPodName + "-external", "default"), "Getting admin server node port failed"); - logger.info("Deploying application {0} in domain {1} cluster target cluster-1", - clusterViewAppPath, domainUid); + logger.info("Deploying application {0} to domain {1} cluster target cluster-1 in namespace {2}", + clusterViewAppPath, domainUid, namespace); ExecResult result = DeployUtil.deployUsingRest(K8S_NODEPORT_HOST, String.valueOf(serviceNodePort), ADMIN_USERNAME_DEFAULT, ADMIN_PASSWORD_DEFAULT, @@ -1291,15 +1362,26 @@ private int getTraefikLbNodePort(boolean isHttps) { "Getting web node port for Traefik loadbalancer failed"); } - private void verifyClusterLoadbalancing(String domainUid, String protocol, int lbPort) { + private void verifyClusterLoadbalancing(String domainUid, String namespace, String protocol, int lbPort, + int replicaCount, boolean hostRouting, String apacheLocationString) { //access application in managed servers through Traefik load balancer logger.info("Accessing the clusterview app through load balancer to verify all servers in cluster"); - String curlRequest = String.format("curl --silent --show-error -ks --noproxy '*' " - + "-H 'host: %s' %s://%s:%s/clusterview/ClusterViewServlet" - + "\"?user=" + ADMIN_USERNAME_DEFAULT - + "&password=" + ADMIN_PASSWORD_DEFAULT + "\"", - domainUid + "." + defaultNamespace + "." + "cluster-1.test", protocol, K8S_NODEPORT_HOST, lbPort); + String curlRequest; + if (hostRouting) { + curlRequest = String.format("curl --silent --show-error -ks --noproxy '*' " + + "-H 'host: %s' %s://%s:%s/clusterview/ClusterViewServlet" + + "\"?user=" + ADMIN_USERNAME_DEFAULT + + "&password=" + ADMIN_PASSWORD_DEFAULT + "\"", + domainUid + "." + namespace + "." + "cluster-1.test", protocol, K8S_NODEPORT_HOST, lbPort); + } else { + curlRequest = String.format("curl --silent --show-error -ks --noproxy '*' " + + "%s://%s:%s/" + apacheLocationString + "/clusterview/ClusterViewServlet" + + "\"?user=" + ADMIN_USERNAME_DEFAULT + + "&password=" + ADMIN_PASSWORD_DEFAULT + "\"", + protocol, K8S_NODEPORT_HOST, lbPort); + } + List managedServers = new ArrayList<>(); for (int i = 1; i <= replicaCount; i++) { managedServers.add(MANAGED_SERVER_NAME_BASE + i); @@ -1308,34 +1390,25 @@ private void verifyClusterLoadbalancing(String domainUid, String protocol, int l // verify each managed server can see other member in the cluster verifyServerCommunication(curlRequest, managedServers); - boolean hostRouting = false; - //access application in managed servers through Traefik load balancer and bind domain in the JNDI tree + boolean containsCorrectDomainUid = false; logger.info("Verifying the requests are routed to correct domain and cluster"); - String curlCmd = String.format("curl --silent --show-error -ks --noproxy '*' " - + "-H 'host: %s' %s://%s:%s/clusterview/ClusterViewServlet" - + "\"?user=" + ADMIN_USERNAME_DEFAULT - + "&password=" + ADMIN_PASSWORD_DEFAULT + "\"", - domainUid + "." + defaultNamespace + "." + "cluster-1.test", protocol, K8S_NODEPORT_HOST, lbPort, domainUid); - - // call the webapp and verify the bound domain name to determine - // the requests are sent to the correct cluster members. for (int i = 0; i < 10; i++) { ExecResult result; try { - logger.info(curlCmd); - result = ExecCommand.exec(curlCmd, true); + logger.info(curlRequest); + result = ExecCommand.exec(curlRequest, true); String response = result.stdout().trim(); logger.info("Response for iteration {0}: exitValue {1}, stdout {2}, stderr {3}", i, result.exitValue(), response, result.stderr()); if (response.contains(domainUid)) { - hostRouting = true; + containsCorrectDomainUid = true; break; } } catch (IOException | InterruptedException ex) { // ignore } } - assertTrue(hostRouting, "Host routing is not working"); + assertTrue(containsCorrectDomainUid, "Host routing is not working"); } /** diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java index e8aa3fa71cb..63419df006a 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java @@ -16,6 +16,7 @@ import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -72,7 +73,6 @@ import static java.nio.file.Files.readString; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; -import static oracle.weblogic.kubernetes.TestConstants.APACHE_IMAGE_12213; import static oracle.weblogic.kubernetes.TestConstants.APACHE_RELEASE_NAME; import static oracle.weblogic.kubernetes.TestConstants.APACHE_SAMPLE_CHART_DIR; import static oracle.weblogic.kubernetes.TestConstants.APPSCODE_REPO_NAME; @@ -616,15 +616,44 @@ public static HelmParams installAndVerifyVoyager(String voyagerNamespace, * Install Apache and wait up to five minutes until the Apache pod is ready. * * @param apacheNamespace the namespace in which the Apache will be installed + * @param image the image name of Apache webtier * @param httpNodePort the http nodeport of Apache * @param httpsNodePort the https nodeport of Apache * @param domainUid the uid of the domain to which Apache will route the services * @return the Apache Helm installation parameters */ public static HelmParams installAndVerifyApache(String apacheNamespace, + String image, int httpNodePort, int httpsNodePort, - String domainUid) { + String domainUid) throws IOException { + return installAndVerifyApache(apacheNamespace, image, httpNodePort, httpsNodePort, domainUid, + null, null, null); + } + + /** + * Install Apache and wait up to five minutes until the Apache pod is ready. + * + * @param apacheNamespace the namespace in which the Apache will be installed + * @param image the image name of Apache webtier + * @param httpNodePort the http nodeport of Apache + * @param httpsNodePort the https nodeport of Apache + * @param domainUid the uid of the domain to which Apache will route the services + * @param volumePath the path to put your own custom_mod_wl_apache.conf file + * @param virtualHostName the VirtualHostName of the Apache HTTP server which is used to enables custom SSL config + * @param clusterNamePortMap the map with clusterName as key and cluster port number as value + * @return the Apache Helm installation parameters + */ + public static HelmParams installAndVerifyApache(String apacheNamespace, + String image, + int httpNodePort, + int httpsNodePort, + String domainUid, + String volumePath, + String virtualHostName, + LinkedHashMap clusterNamePortMap) + throws IOException { + LoggingFacade logger = getLogger(); // Create Docker registry secret in the operator namespace to pull the image from repository @@ -647,7 +676,7 @@ public static HelmParams installAndVerifyApache(String apacheNamespace, ApacheParams apacheParams = new ApacheParams() .helmParams(apacheHelmParams) .imagePullSecrets(secretNameMap) - .image(APACHE_IMAGE_12213) + .image(image) .domainUID(domainUid); if (httpNodePort >= 0 && httpsNodePort >= 0) { @@ -656,6 +685,70 @@ public static HelmParams installAndVerifyApache(String apacheNamespace, .httpsNodePort(httpsNodePort); } + if (volumePath != null && clusterNamePortMap != null) { + // create a custom Apache plugin configuration file named custom_mod_wl_apache.conf + Path customConf = Paths.get(volumePath, "custom_mod_wl_apache.conf"); + ArrayList lines = new ArrayList<>(); + lines.add(""); + lines.add("WebLogicHost ${WEBLOGIC_HOST}"); + lines.add("WebLogicPort ${WEBLOGIC_PORT}"); + lines.add(""); + + lines.add(""); + lines.add("SetHandler weblogic-handler"); + lines.add("WebLogicHost " + domainUid + "-admin-server"); + lines.add("WebLogicPort ${WEBLOGIC_PORT}"); + lines.add(""); + + int i = 1; + for (String clusterName : clusterNamePortMap.keySet()) { + lines.add(""); + lines.add("WLSRequest On"); + lines.add("WebLogicCluster " + clusterName + ":" + clusterNamePortMap.get(clusterName)); + lines.add("PathTrim /weblogic" + i); + lines.add(""); + i++; + } + + try { + Files.write(customConf, lines); + } catch (IOException ioex) { + logger.info("Got IOException while write to a file"); + throw ioex; + } + + apacheParams.volumePath(volumePath); + } + + if (virtualHostName != null) { + // create the certificate and private key + String certFile = RESULTS_ROOT + "/apache-sample.crt"; + String keyFile = RESULTS_ROOT + "/apache-sample.key"; + + Map envs = new HashMap<>(); + envs.put("VIRTUAL_HOST_NAME", virtualHostName); + envs.put("SSL_CERT_FILE", certFile); + envs.put("SSL_CERT_KEY_FILE", keyFile); + + String command = "sh ../kubernetes/samples/charts/apache-samples/custom-sample/certgen.sh"; + CommandParams params = Command + .defaultCommandParams() + .command(command) + .env(envs) + .saveResults(true) + .redirect(true); + + Command.withParams(params).execute(); + + String customCert = Base64.getEncoder().encodeToString(Files.readAllBytes(Paths.get(certFile))); + String customKey = Base64.getEncoder().encodeToString(Files.readAllBytes(Paths.get(keyFile))); + + // set the Apache helm install parameters + apacheParams.virtualHostName(virtualHostName); + apacheParams.customCert(customCert); + apacheParams.customKey(customKey); + } + // install Apache assertThat(installApache(apacheParams)) .as("Test Apache installation succeeds") From aab305c7ec261f594d18f5da4399df97ea9b5d0f Mon Sep 17 00:00:00 2001 From: xiancao Date: Wed, 26 Aug 2020 19:20:30 +0000 Subject: [PATCH 04/15] fix apache-webtier chart imagePullSecret --- .../samples/charts/apache-webtier/templates/deployment.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kubernetes/samples/charts/apache-webtier/templates/deployment.yaml b/kubernetes/samples/charts/apache-webtier/templates/deployment.yaml index ec7622f1708..4cdf9722c2d 100644 --- a/kubernetes/samples/charts/apache-webtier/templates/deployment.yaml +++ b/kubernetes/samples/charts/apache-webtier/templates/deployment.yaml @@ -33,6 +33,10 @@ spec: - name: {{ template "apache.fullname" . }} hostPath: path: {{ .Values.volumePath | quote }} +{{- end }} +{{- if .Values.imagePullSecrets }} + imagePullSecrets: + - name: {{ (index .Values.imagePullSecrets 0).name }} {{- end }} containers: - name: {{ template "apache.fullname" . }} From f9c510964fecb378c522910eb17e5f686cdc9048 Mon Sep 17 00:00:00 2001 From: xiancao Date: Wed, 26 Aug 2020 22:32:50 +0000 Subject: [PATCH 05/15] get the ItPodsShutdownOption from shutdown3 branch --- .../oracle/weblogic/kubernetes/ItPodsShutdownOption.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItPodsShutdownOption.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItPodsShutdownOption.java index 8f89a22dc79..f02cade869f 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItPodsShutdownOption.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItPodsShutdownOption.java @@ -59,7 +59,7 @@ */ @DisplayName("Verify shutdown rules when shutdown properties are defined at different levels") @IntegrationTest -class ItPodsShutdown { +class ItPodsShutdownOption { private static String domainNamespace = null; @@ -102,8 +102,8 @@ public static void initAll(@Namespaces(2) List namespaces) { * This test is to verify different shutdown options specified at different scopes in Domain Resource Definition. * Refer to section "Shutdown options" of Documentation link: * https://oracle.github.io/weblogic-kubernetes-operator/userguide/managing-domains/domain-lifecycle/startup/ - * step 1: Startup a WebLogic domain with one cluster that initially has one running managed server. The shutdown option is - * configured as follows: + * step 1: Startup a WebLogic domain with one cluster that initially has one running managed server. The shutdown + * option is configured as follows: * domain: SHUTDOWN_TYPE -> Graceful. * adminServer: SHUTDOWN_TYPE -> Forced. * SHUTDOWN_IGNORE_SESSIONS -> true. From f56c7d27300bc92ad3911d487c5519c7869c3eef Mon Sep 17 00:00:00 2001 From: xiancao Date: Thu, 27 Aug 2020 05:49:57 +0000 Subject: [PATCH 06/15] change imagePullPolicy to Always --- .../java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java | 1 + 1 file changed, 1 insertion(+) diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java index 63419df006a..47d0aee992a 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java @@ -677,6 +677,7 @@ public static HelmParams installAndVerifyApache(String apacheNamespace, .helmParams(apacheHelmParams) .imagePullSecrets(secretNameMap) .image(image) + .imagePullPolicy("Always") .domainUID(domainUid); if (httpNodePort >= 0 && httpsNodePort >= 0) { From 5c6d6bd024e6a1f18a0d8f0ff8a6650a242eb452 Mon Sep 17 00:00:00 2001 From: xiancao Date: Thu, 27 Aug 2020 20:33:32 +0000 Subject: [PATCH 07/15] push the apache image to Kind repo --- .../kubernetes/ItTwoDomainsLoadBalancers.java | 50 ++++++++++++++++++- .../kubernetes/extensions/ImageBuilders.java | 5 +- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java index 04a383cb450..9f4a66b9c89 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import com.gargoylesoftware.htmlunit.WebClient; @@ -93,6 +94,9 @@ import static oracle.weblogic.kubernetes.TestConstants.OCR_REGISTRY; import static oracle.weblogic.kubernetes.TestConstants.OCR_SECRET_NAME; import static oracle.weblogic.kubernetes.TestConstants.PV_ROOT; +import static oracle.weblogic.kubernetes.TestConstants.REPO_PASSWORD; +import static oracle.weblogic.kubernetes.TestConstants.REPO_REGISTRY; +import static oracle.weblogic.kubernetes.TestConstants.REPO_USERNAME; import static oracle.weblogic.kubernetes.TestConstants.RESULTS_ROOT; import static oracle.weblogic.kubernetes.TestConstants.VOYAGER_CHART_NAME; import static oracle.weblogic.kubernetes.actions.ActionConstants.APP_DIR; @@ -107,6 +111,10 @@ import static oracle.weblogic.kubernetes.actions.TestActions.deletePersistentVolumeClaim; import static oracle.weblogic.kubernetes.actions.TestActions.deletePod; import static oracle.weblogic.kubernetes.actions.TestActions.deleteSecret; +import static oracle.weblogic.kubernetes.actions.TestActions.dockerLogin; +import static oracle.weblogic.kubernetes.actions.TestActions.dockerPull; +import static oracle.weblogic.kubernetes.actions.TestActions.dockerPush; +import static oracle.weblogic.kubernetes.actions.TestActions.dockerTag; import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort; import static oracle.weblogic.kubernetes.actions.TestActions.listIngresses; import static oracle.weblogic.kubernetes.actions.TestActions.listJobs; @@ -182,6 +190,7 @@ public class ItTwoDomainsLoadBalancers { private static Path dstFile = null; private static HelmParams apacheHelmParams1 = null; private static HelmParams apacheHelmParams2 = null; + private static String kindRepoApacheImage = APACHE_IMAGE_12213; // domain constants private final String clusterName = "cluster-1"; @@ -246,6 +255,33 @@ public static void initAll(@Namespaces(6) List namespaces) { logger.info("Using image {0}", kindRepoImage); image = kindRepoImage; isUseSecret = false; + + // We do not know why the kind clusters can't pull Apache webtier image from OCIR using the image pull secret. + // Try the following instead + // 1. docker login + // 2. docker pull + // 3. docker tag with the KIND_REPO value + ConditionFactory withStandardRetryPolicy + = with().pollDelay(0, SECONDS) + .and().with().pollInterval(10, SECONDS) + .atMost(30, MINUTES).await(); + + withStandardRetryPolicy + .conditionEvaluationListener( + condition -> logger.info("Waiting for docker login to be successful" + + "(elapsed time {0} ms, remaining time {1} ms)", + condition.getElapsedTimeInMS(), + condition.getRemainingTimeInMS())) + .until(() -> dockerLogin(REPO_REGISTRY, REPO_USERNAME, REPO_PASSWORD)); + + withStandardRetryPolicy + .conditionEvaluationListener( + condition -> logger.info("Waiting for pullImageFromOcirAndPushToKind for image {0} to be successful" + + "(elapsed time {1} ms, remaining time {2} ms)", APACHE_IMAGE_12213, + condition.getElapsedTimeInMS(), + condition.getRemainingTimeInMS())) + .until(pullImageFromOcirAndPushToKind(APACHE_IMAGE_12213) + ); } } @@ -356,14 +392,14 @@ public void testDeployAppAndInstallIngressControllers() { // install and verify Apache apacheHelmParams1 = assertDoesNotThrow( - () -> installAndVerifyApache(domain1Namespace, APACHE_IMAGE_12213, 0, 0, domain1Uid)); + () -> installAndVerifyApache(domain1Namespace, kindRepoApacheImage, 0, 0, domain1Uid)); LinkedHashMap clusterNamePortMap = new LinkedHashMap<>(); for (int i = 0; i < numberOfDomains; i++) { clusterNamePortMap.put(domainUids.get(i) + "-cluster-cluster-1", managedServerPort); } apacheHelmParams2 = assertDoesNotThrow( - () -> installAndVerifyApache(defaultNamespace, APACHE_IMAGE_12213, 0, 0, domain1Uid, + () -> installAndVerifyApache(defaultNamespace, kindRepoApacheImage, 0, 0, domain1Uid, RESULTS_ROOT, "apache-sample-host", clusterNamePortMap)); } @@ -1452,4 +1488,14 @@ private static boolean adminNodePortAccessible(int nodePort, String userName, St return adminAccessible; } + + private static Callable pullImageFromOcirAndPushToKind(String apacheImage) { + return (() -> { + kindRepoApacheImage = KIND_REPO + apacheImage.substring(TestConstants.REPO_DEFAULT.length()); + logger.info("pulling image {0} from ocir, tag it as image {1} and push to kind repo", + apacheImage, kindRepoApacheImage); + return dockerPull(apacheImage) && dockerTag(apacheImage, kindRepoApacheImage) && dockerPush(kindRepoApacheImage); + }); + } + } diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/ImageBuilders.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/ImageBuilders.java index f52ef6f821e..a8745cd42b1 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/ImageBuilders.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/ImageBuilders.java @@ -48,6 +48,8 @@ import static oracle.weblogic.kubernetes.TestConstants.OCR_USERNAME; import static oracle.weblogic.kubernetes.TestConstants.REPO_DUMMY_VALUE; import static oracle.weblogic.kubernetes.TestConstants.REPO_NAME; +import static oracle.weblogic.kubernetes.TestConstants.REPO_PASSWORD; +import static oracle.weblogic.kubernetes.TestConstants.REPO_REGISTRY; import static oracle.weblogic.kubernetes.TestConstants.REPO_USERNAME; import static oracle.weblogic.kubernetes.TestConstants.WDT_BASIC_APP_NAME; import static oracle.weblogic.kubernetes.TestConstants.WDT_BASIC_IMAGE_DOMAINHOME; @@ -178,9 +180,10 @@ public void beforeAll(ExtensionContext context) { + "(elapsed time {0} ms, remaining time {1} ms)", condition.getElapsedTimeInMS(), condition.getRemainingTimeInMS())) - .until(() -> dockerLogin(OCR_REGISTRY, OCR_USERNAME, OCR_PASSWORD)); + .until(() -> dockerLogin(REPO_REGISTRY, REPO_USERNAME, REPO_PASSWORD)); } } + // push the images to repo if (!REPO_NAME.isEmpty()) { From ac942b8a87db6bfdbef5f65901dcdcbddda73ceb Mon Sep 17 00:00:00 2001 From: xiancao Date: Thu, 27 Aug 2020 21:50:07 +0000 Subject: [PATCH 08/15] add debug info for REPO_REGISTRY etc in Kind new --- .../weblogic/kubernetes/ItTwoDomainsLoadBalancers.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java index 9f4a66b9c89..2e812bd7f4c 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java @@ -91,8 +91,10 @@ import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST; import static oracle.weblogic.kubernetes.TestConstants.KIND_REPO; import static oracle.weblogic.kubernetes.TestConstants.MANAGED_SERVER_NAME_BASE; +import static oracle.weblogic.kubernetes.TestConstants.OCR_PASSWORD; import static oracle.weblogic.kubernetes.TestConstants.OCR_REGISTRY; import static oracle.weblogic.kubernetes.TestConstants.OCR_SECRET_NAME; +import static oracle.weblogic.kubernetes.TestConstants.OCR_USERNAME; import static oracle.weblogic.kubernetes.TestConstants.PV_ROOT; import static oracle.weblogic.kubernetes.TestConstants.REPO_PASSWORD; import static oracle.weblogic.kubernetes.TestConstants.REPO_REGISTRY; @@ -266,6 +268,13 @@ public static void initAll(@Namespaces(6) List namespaces) { .and().with().pollInterval(10, SECONDS) .atMost(30, MINUTES).await(); + logger.info("DEBUG: REPO_REGISTRY: {0}", REPO_REGISTRY); + logger.info("DEBUG: REPO_USERNAME: {0}", REPO_USERNAME); + logger.info("DEBUG: REPO_PASSWORD: {0}", REPO_PASSWORD); + logger.info("DEBUG: OCR_REGISTRY: {0}", OCR_REGISTRY); + logger.info("DEBUG: OCR_USERNAME: {0}", OCR_USERNAME); + logger.info("DEBUG: OCR_PASSWORD: {0}", OCR_PASSWORD); + withStandardRetryPolicy .conditionEvaluationListener( condition -> logger.info("Waiting for docker login to be successful" From 5ab3b3d353b7d7ee419b50f091be7cad6cd4b373 Mon Sep 17 00:00:00 2001 From: xiancao Date: Thu, 27 Aug 2020 22:35:25 +0000 Subject: [PATCH 09/15] remoe pull image from ocir and push to kind repo --- .../kubernetes/ItTwoDomainsLoadBalancers.java | 14 ++++++++------ .../oracle/weblogic/kubernetes/TestConstants.java | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java index 2e812bd7f4c..7dcec9f86f4 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java @@ -91,14 +91,14 @@ import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST; import static oracle.weblogic.kubernetes.TestConstants.KIND_REPO; import static oracle.weblogic.kubernetes.TestConstants.MANAGED_SERVER_NAME_BASE; -import static oracle.weblogic.kubernetes.TestConstants.OCR_PASSWORD; +//import static oracle.weblogic.kubernetes.TestConstants.OCR_PASSWORD; import static oracle.weblogic.kubernetes.TestConstants.OCR_REGISTRY; import static oracle.weblogic.kubernetes.TestConstants.OCR_SECRET_NAME; -import static oracle.weblogic.kubernetes.TestConstants.OCR_USERNAME; +//import static oracle.weblogic.kubernetes.TestConstants.OCR_USERNAME; import static oracle.weblogic.kubernetes.TestConstants.PV_ROOT; -import static oracle.weblogic.kubernetes.TestConstants.REPO_PASSWORD; -import static oracle.weblogic.kubernetes.TestConstants.REPO_REGISTRY; -import static oracle.weblogic.kubernetes.TestConstants.REPO_USERNAME; +//import static oracle.weblogic.kubernetes.TestConstants.REPO_PASSWORD; +//import static oracle.weblogic.kubernetes.TestConstants.REPO_REGISTRY; +//import static oracle.weblogic.kubernetes.TestConstants.REPO_USERNAME; import static oracle.weblogic.kubernetes.TestConstants.RESULTS_ROOT; import static oracle.weblogic.kubernetes.TestConstants.VOYAGER_CHART_NAME; import static oracle.weblogic.kubernetes.actions.ActionConstants.APP_DIR; @@ -113,7 +113,7 @@ import static oracle.weblogic.kubernetes.actions.TestActions.deletePersistentVolumeClaim; import static oracle.weblogic.kubernetes.actions.TestActions.deletePod; import static oracle.weblogic.kubernetes.actions.TestActions.deleteSecret; -import static oracle.weblogic.kubernetes.actions.TestActions.dockerLogin; +//import static oracle.weblogic.kubernetes.actions.TestActions.dockerLogin; import static oracle.weblogic.kubernetes.actions.TestActions.dockerPull; import static oracle.weblogic.kubernetes.actions.TestActions.dockerPush; import static oracle.weblogic.kubernetes.actions.TestActions.dockerTag; @@ -263,6 +263,7 @@ public static void initAll(@Namespaces(6) List namespaces) { // 1. docker login // 2. docker pull // 3. docker tag with the KIND_REPO value + /* ConditionFactory withStandardRetryPolicy = with().pollDelay(0, SECONDS) .and().with().pollInterval(10, SECONDS) @@ -291,6 +292,7 @@ public static void initAll(@Namespaces(6) List namespaces) { condition.getRemainingTimeInMS())) .until(pullImageFromOcirAndPushToKind(APACHE_IMAGE_12213) ); + */ } } diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java index 7758c0ffa96..64e48995aef 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java @@ -95,7 +95,7 @@ public interface TestConstants { public static final String APPSCODE_REPO_NAME = "appscode"; public static final String VOYAGER_CHART_NAME = "voyager"; public static final String VOYAGER_CHART_VERSION = "12.0.0"; - + // Apache constants public static final String APACHE_RELEASE_NAME = "apache-release" + BUILD_ID; public static final String APACHE_SAMPLE_CHART_DIR = "../kubernetes/samples/charts/apache-webtier"; From 16d521843fded7b1f6d38e7e2589d663be17d4c8 Mon Sep 17 00:00:00 2001 From: xiancao Date: Fri, 28 Aug 2020 14:54:44 +0000 Subject: [PATCH 10/15] enable pull apache image from ocir and push to kind new --- .../kubernetes/ItTwoDomainsLoadBalancers.java | 26 ++++++------------- 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java index 7dcec9f86f4..b39840aa0eb 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java @@ -91,14 +91,12 @@ import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST; import static oracle.weblogic.kubernetes.TestConstants.KIND_REPO; import static oracle.weblogic.kubernetes.TestConstants.MANAGED_SERVER_NAME_BASE; -//import static oracle.weblogic.kubernetes.TestConstants.OCR_PASSWORD; import static oracle.weblogic.kubernetes.TestConstants.OCR_REGISTRY; import static oracle.weblogic.kubernetes.TestConstants.OCR_SECRET_NAME; -//import static oracle.weblogic.kubernetes.TestConstants.OCR_USERNAME; import static oracle.weblogic.kubernetes.TestConstants.PV_ROOT; -//import static oracle.weblogic.kubernetes.TestConstants.REPO_PASSWORD; -//import static oracle.weblogic.kubernetes.TestConstants.REPO_REGISTRY; -//import static oracle.weblogic.kubernetes.TestConstants.REPO_USERNAME; +import static oracle.weblogic.kubernetes.TestConstants.REPO_PASSWORD; +import static oracle.weblogic.kubernetes.TestConstants.REPO_REGISTRY; +import static oracle.weblogic.kubernetes.TestConstants.REPO_USERNAME; import static oracle.weblogic.kubernetes.TestConstants.RESULTS_ROOT; import static oracle.weblogic.kubernetes.TestConstants.VOYAGER_CHART_NAME; import static oracle.weblogic.kubernetes.actions.ActionConstants.APP_DIR; @@ -113,7 +111,7 @@ import static oracle.weblogic.kubernetes.actions.TestActions.deletePersistentVolumeClaim; import static oracle.weblogic.kubernetes.actions.TestActions.deletePod; import static oracle.weblogic.kubernetes.actions.TestActions.deleteSecret; -//import static oracle.weblogic.kubernetes.actions.TestActions.dockerLogin; +import static oracle.weblogic.kubernetes.actions.TestActions.dockerLogin; import static oracle.weblogic.kubernetes.actions.TestActions.dockerPull; import static oracle.weblogic.kubernetes.actions.TestActions.dockerPush; import static oracle.weblogic.kubernetes.actions.TestActions.dockerTag; @@ -263,19 +261,13 @@ public static void initAll(@Namespaces(6) List namespaces) { // 1. docker login // 2. docker pull // 3. docker tag with the KIND_REPO value - /* + // 4. docker push to KIND_REPO + ConditionFactory withStandardRetryPolicy = with().pollDelay(0, SECONDS) .and().with().pollInterval(10, SECONDS) .atMost(30, MINUTES).await(); - logger.info("DEBUG: REPO_REGISTRY: {0}", REPO_REGISTRY); - logger.info("DEBUG: REPO_USERNAME: {0}", REPO_USERNAME); - logger.info("DEBUG: REPO_PASSWORD: {0}", REPO_PASSWORD); - logger.info("DEBUG: OCR_REGISTRY: {0}", OCR_REGISTRY); - logger.info("DEBUG: OCR_USERNAME: {0}", OCR_USERNAME); - logger.info("DEBUG: OCR_PASSWORD: {0}", OCR_PASSWORD); - withStandardRetryPolicy .conditionEvaluationListener( condition -> logger.info("Waiting for docker login to be successful" @@ -290,9 +282,7 @@ public static void initAll(@Namespaces(6) List namespaces) { + "(elapsed time {1} ms, remaining time {2} ms)", APACHE_IMAGE_12213, condition.getElapsedTimeInMS(), condition.getRemainingTimeInMS())) - .until(pullImageFromOcirAndPushToKind(APACHE_IMAGE_12213) - ); - */ + .until(pullImageFromOcirAndPushToKind(APACHE_IMAGE_12213)); } } @@ -411,7 +401,7 @@ public void testDeployAppAndInstallIngressControllers() { } apacheHelmParams2 = assertDoesNotThrow( () -> installAndVerifyApache(defaultNamespace, kindRepoApacheImage, 0, 0, domain1Uid, - RESULTS_ROOT, "apache-sample-host", clusterNamePortMap)); + PV_ROOT + "/" + this.getClass().getSimpleName(), "apache-sample-host", clusterNamePortMap)); } /** From f1294ffa95f12e56359729a780d2a3bc975d0c64 Mon Sep 17 00:00:00 2001 From: xiancao Date: Fri, 28 Aug 2020 19:15:51 +0000 Subject: [PATCH 11/15] cleanup --- .../kubernetes/ItTwoDomainsLoadBalancers.java | 75 ++++++---- .../weblogic/kubernetes/TestConstants.java | 4 +- .../kubernetes/actions/impl/Apache.java | 133 +----------------- .../kubernetes/actions/impl/ApacheParams.java | 3 +- .../kubernetes/assertions/TestAssertions.java | 7 +- .../kubernetes/assertions/impl/Apache.java | 2 +- .../assertions/impl/Kubernetes.java | 4 +- .../kubernetes/utils/CommonTestUtils.java | 12 +- 8 files changed, 61 insertions(+), 179 deletions(-) diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java index b39840aa0eb..ce3d734575c 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java @@ -94,6 +94,7 @@ import static oracle.weblogic.kubernetes.TestConstants.OCR_REGISTRY; import static oracle.weblogic.kubernetes.TestConstants.OCR_SECRET_NAME; import static oracle.weblogic.kubernetes.TestConstants.PV_ROOT; +import static oracle.weblogic.kubernetes.TestConstants.REPO_DEFAULT; import static oracle.weblogic.kubernetes.TestConstants.REPO_PASSWORD; import static oracle.weblogic.kubernetes.TestConstants.REPO_REGISTRY; import static oracle.weblogic.kubernetes.TestConstants.REPO_USERNAME; @@ -432,7 +433,7 @@ public void testTraefikHttpHostRoutingAcrossDomains() { // verify load balancing works when 2 domains are running in the same namespace logger.info("Verifying http traffic"); for (String domainUid : domainUids) { - verifyClusterLoadbalancing(domainUid, defaultNamespace,"http", getTraefikLbNodePort(false), + verifyClusterLoadbalancing(domainUid, defaultNamespace, "http", getTraefikLbNodePort(false), replicaCount, true, ""); } } @@ -474,26 +475,25 @@ public void testVoyagerHostHttpRoutingAcrossDomains() { } /** - * Verify Apache load balancer default sample. + * Verify Apache load balancer default sample through HTTP channel. */ @Order(8) @Test - @DisplayName("verify Apache loadbalancer default sample through HTTP channel") + @DisplayName("verify Apache load balancer default sample through HTTP channel") public void testApacheDefaultSample() { // verify Apache default sample logger.info("Verifying Apache default sample"); int httpNodePort = getApacheNodePort(domain1Namespace, "http"); - verifyClusterLoadbalancing(domain1Uid, domain1Namespace, "http", httpNodePort, 3, - false, "weblogic"); + verifyClusterLoadbalancing(domain1Uid, domain1Namespace, "http", httpNodePort, 3, false, "/weblogic"); } /** - * Verify Apache load balancer custom sample. + * Verify Apache load balancer custom sample through HTTP and HTTPS channel. */ @Order(9) @Test - @DisplayName("verify Apache loadbalancer custom sample through HTTP and HTTPS channel") + @DisplayName("verify Apache load balancer custom sample through HTTP and HTTPS channel") public void testApacheCustomSample() { // verify Apache custom sample @@ -501,11 +501,11 @@ public void testApacheCustomSample() { for (int i = 1; i <= numberOfDomains; i++) { int httpNodePort = getApacheNodePort(defaultNamespace, "http"); verifyClusterLoadbalancing(domainUids.get(i - 1), defaultNamespace, "http", httpNodePort, replicaCount, - false, "weblogic" + i); + false, "/weblogic" + i); int httpsNodePort = getApacheNodePort(defaultNamespace, "https"); verifyClusterLoadbalancing(domainUids.get(i - 1), defaultNamespace, "https", httpsNodePort, replicaCount, - false, "weblogic" + i); + false, "/weblogic" + i); } } @@ -533,15 +533,15 @@ public void tearDownAll() { // uninstall Apache if (apacheHelmParams1 != null) { assertThat(uninstallApache(apacheHelmParams1)) - .as("Test uninstallApache returns true") - .withFailMessage("uninstallApache() did not return true") + .as("Test whether uninstallApache in domain1Namespace returns true") + .withFailMessage("uninstallApache() in domain1Namespace did not return true") .isTrue(); } if (apacheHelmParams2 != null) { assertThat(uninstallApache(apacheHelmParams2)) - .as("Test uninstallApache returns true") - .withFailMessage("uninstallApache() did not return true") + .as("Test whether uninstallApache in default namespace returns true") + .withFailMessage("uninstallApache() in default namespace did not return true") .isTrue(); } @@ -1023,7 +1023,7 @@ private void getBothDomainsPodsOriginalCreationTimestamp(List domainUids * Shutdown domain and verify all the server pods were shutdown. * * @param domainNamespace the namespace where the domain exists - * @param domainUid the uid of the domain + * @param domainUid the uid of the domain to shutdown */ private void shutdownDomainAndVerify(String domainNamespace, String domainUid) { @@ -1037,8 +1037,8 @@ private void shutdownDomainAndVerify(String domainNamespace, String domainUid) { checkPodDoesNotExist(domainUid + "-" + ADMIN_SERVER_NAME_BASE, domainUid, domainNamespace); - for (int j = 1; j <= replicaCount; j++) { - String managedServerPodName = domainUid + "-" + MANAGED_SERVER_NAME_BASE + j; + for (int i = 1; i <= replicaCount; i++) { + String managedServerPodName = domainUid + "-" + MANAGED_SERVER_NAME_BASE + i; checkPodDoesNotExist(managedServerPodName, domainUid, domainNamespace); } } @@ -1333,12 +1333,12 @@ private static int getVoyagerLbNodePort(String ingressName) { private static int getApacheNodePort(String namespace, String channelName) { String apacheServiceName = APACHE_RELEASE_NAME + "-" + namespace.substring(3) + "-apache-webtier"; - // get ingress service Nodeport - int apacheHttpNodePort = assertDoesNotThrow(() -> + // get Apache service NodePort + int apacheNodePort = assertDoesNotThrow(() -> getServiceNodePort(namespace, apacheServiceName, channelName), - "Getting Apache load balancer service node port failed"); - logger.info("Node port for {0} is: {1} :", apacheServiceName, apacheHttpNodePort); - return apacheHttpNodePort; + "Getting Apache service NodePort failed"); + logger.info("NodePort for {0} is: {1} :", apacheServiceName, apacheNodePort); + return apacheNodePort; } private static void deployApplication(String namespace, String domainUid, String adminServerPodName) { @@ -1399,10 +1399,26 @@ private int getTraefikLbNodePort(boolean isHttps) { "Getting web node port for Traefik loadbalancer failed"); } - private void verifyClusterLoadbalancing(String domainUid, String namespace, String protocol, int lbPort, - int replicaCount, boolean hostRouting, String apacheLocationString) { - - //access application in managed servers through Traefik load balancer + /** + * Verify cluster load balancing with ClusterViewServlet app. + * + * @param domainUid uid of the domain in which the cluster exists + * @param namespace namespace in which the domain exists + * @param protocol protocol used to test, accepted value: http or https + * @param lbPort load balancer service port + * @param replicaCount replica count of the managed servers in the cluster + * @param hostRouting whether it is a host base routing + * @param apacheLocationString location string in apache configuration + */ + private void verifyClusterLoadbalancing(String domainUid, + String namespace, + String protocol, + int lbPort, + int replicaCount, + boolean hostRouting, + String apacheLocationString) { + + // access application in managed servers through load balancer logger.info("Accessing the clusterview app through load balancer to verify all servers in cluster"); String curlRequest; if (hostRouting) { @@ -1413,7 +1429,7 @@ private void verifyClusterLoadbalancing(String domainUid, String namespace, Stri domainUid + "." + namespace + "." + "cluster-1.test", protocol, K8S_NODEPORT_HOST, lbPort); } else { curlRequest = String.format("curl --silent --show-error -ks --noproxy '*' " - + "%s://%s:%s/" + apacheLocationString + "/clusterview/ClusterViewServlet" + + "%s://%s:%s" + apacheLocationString + "/clusterview/ClusterViewServlet" + "\"?user=" + ADMIN_USERNAME_DEFAULT + "&password=" + ADMIN_PASSWORD_DEFAULT + "\"", protocol, K8S_NODEPORT_HOST, lbPort); @@ -1445,7 +1461,7 @@ private void verifyClusterLoadbalancing(String domainUid, String namespace, Stri // ignore } } - assertTrue(containsCorrectDomainUid, "Host routing is not working"); + assertTrue(containsCorrectDomainUid, "The request was not routed to the correct domain"); } /** @@ -1492,11 +1508,10 @@ private static boolean adminNodePortAccessible(int nodePort, String userName, St private static Callable pullImageFromOcirAndPushToKind(String apacheImage) { return (() -> { - kindRepoApacheImage = KIND_REPO + apacheImage.substring(TestConstants.REPO_DEFAULT.length()); - logger.info("pulling image {0} from ocir, tag it as image {1} and push to kind repo", + kindRepoApacheImage = KIND_REPO + apacheImage.substring(REPO_DEFAULT.length()); + logger.info("pulling image {0} from OCIR, tag it as image {1} and push to KIND repo", apacheImage, kindRepoApacheImage); return dockerPull(apacheImage) && dockerTag(apacheImage, kindRepoApacheImage) && dockerPush(kindRepoApacheImage); }); } - } diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java index 64e48995aef..4b97420f710 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java @@ -80,9 +80,6 @@ public interface TestConstants { public static final String STABLE_REPO_NAME = "stable"; public static final String NGINX_CHART_NAME = "nginx-ingress"; - // Apache constants - public static final String APACHE_IMAGE_12213 = "phx.ocir.io/weblogick8s/oracle/apache:12.2.1.3"; - // Traefik constants public static final String TRAEFIK_REPO_URL = "https://containous.github.io/traefik-helm-chart"; public static final String TRAEFIK_REPO_NAME = "traefik"; @@ -97,6 +94,7 @@ public interface TestConstants { public static final String VOYAGER_CHART_VERSION = "12.0.0"; // Apache constants + public static final String APACHE_IMAGE_12213 = "phx.ocir.io/weblogick8s/oracle/apache:12.2.1.3"; public static final String APACHE_RELEASE_NAME = "apache-release" + BUILD_ID; public static final String APACHE_SAMPLE_CHART_DIR = "../kubernetes/samples/charts/apache-webtier"; diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/Apache.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/Apache.java index ab73af5337e..1ec8b012bfd 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/Apache.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/Apache.java @@ -3,36 +3,14 @@ package oracle.weblogic.kubernetes.actions.impl; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import io.kubernetes.client.custom.IntOrString; -import io.kubernetes.client.openapi.ApiException; -import io.kubernetes.client.openapi.models.NetworkingV1beta1HTTPIngressPath; -import io.kubernetes.client.openapi.models.NetworkingV1beta1HTTPIngressRuleValue; -import io.kubernetes.client.openapi.models.NetworkingV1beta1Ingress; -import io.kubernetes.client.openapi.models.NetworkingV1beta1IngressBackend; -import io.kubernetes.client.openapi.models.NetworkingV1beta1IngressList; -import io.kubernetes.client.openapi.models.NetworkingV1beta1IngressRule; -import io.kubernetes.client.openapi.models.NetworkingV1beta1IngressSpec; -import io.kubernetes.client.openapi.models.V1ObjectMeta; import oracle.weblogic.kubernetes.actions.impl.primitive.Helm; import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams; -import oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes; - -import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; /** - * Utility class for Apache ingress controller. + * Utility class for Apache load balancer. */ public class Apache { - private static final String INGRESS_API_VERSION = "networking.k8s.io/v1beta1"; - private static final String INGRESS_KIND = "Ingress"; - private static final String INGRESS_NGINX_CLASS = "nginx"; - /** * Install Apache Helm chart. * @@ -64,113 +42,4 @@ public static boolean uninstall(HelmParams params) { return Helm.uninstall(params); } - /** - * Create an ingress for the WebLogic domain with domainUid in the specified domain namespace. - * The ingress host is set to 'domainUid.clusterName.test'. - * - * @param ingressName name of the ingress to be created - * @param domainNamespace the WebLogic domain namespace in which the ingress will be created - * @param domainUid the WebLogic domainUid which is backend to the ingress - * @param clusterNameMsPortMap the map with key as cluster name and value as managed server port of the cluster - * @return list of ingress hosts or null if got ApiException when calling Kubernetes client API to create ingress - */ - public static List createIngress(String ingressName, - String domainNamespace, - String domainUid, - Map clusterNameMsPortMap) { - return createIngress(ingressName, domainNamespace, domainUid, clusterNameMsPortMap, true); - } - - /** - * Create an ingress for the WebLogic domain with domainUid in the specified domain namespace. - * The ingress host is set to 'domainUid.domainNamespace.clusterName.test'. - * - * @param ingressName name of the ingress to be created - * @param domainNamespace the WebLogic domain namespace in which the ingress will be created - * @param domainUid the WebLogic domainUid which is backend to the ingress - * @param clusterNameMsPortMap the map with key as cluster name and value as managed server port of the cluster - * @param setIngressHost if false will set to any - * @return list of ingress hosts or null if got ApiException when calling Kubernetes client API to create ingress - */ - public static List createIngress(String ingressName, - String domainNamespace, - String domainUid, - Map clusterNameMsPortMap, - boolean setIngressHost) { - - // set the annotation for kubernetes.io/ingress.class to "nginx" - HashMap annotation = new HashMap<>(); - annotation.put("kubernetes.io/ingress.class", INGRESS_NGINX_CLASS); - - List ingressHostList = new ArrayList<>(); - ArrayList ingressRules = new ArrayList<>(); - clusterNameMsPortMap.forEach((clusterName, managedServerPort) -> { - // set the http ingress paths - NetworkingV1beta1HTTPIngressPath httpIngressPath = new NetworkingV1beta1HTTPIngressPath() - .path(null) - .backend(new NetworkingV1beta1IngressBackend() - .serviceName(domainUid + "-cluster-" + clusterName.toLowerCase().replace("_", "-")) - .servicePort(new IntOrString(managedServerPort)) - ); - ArrayList httpIngressPaths = new ArrayList<>(); - httpIngressPaths.add(httpIngressPath); - - // set the ingress rule - String ingressHost = domainUid + "." + domainNamespace + "." + clusterName + ".test"; - if (!setIngressHost) { - ingressHost = ""; - ingressHostList.add("*"); - } - NetworkingV1beta1IngressRule ingressRule = new NetworkingV1beta1IngressRule() - .host(ingressHost) - .http(new NetworkingV1beta1HTTPIngressRuleValue() - .paths(httpIngressPaths)); - - ingressRules.add(ingressRule); - ingressHostList.add(ingressHost); - - }); - - // set the ingress - NetworkingV1beta1Ingress ingress = new NetworkingV1beta1Ingress() - .apiVersion(INGRESS_API_VERSION) - .kind(INGRESS_KIND) - .metadata(new V1ObjectMeta() - .name(ingressName) - .namespace(domainNamespace) - .annotations(annotation)) - .spec(new NetworkingV1beta1IngressSpec() - .rules(ingressRules)); - - // create the ingress - try { - Kubernetes.createIngress(domainNamespace, ingress); - } catch (ApiException apex) { - getLogger().severe("got ApiException while calling createIngress: {0}", apex.getResponseBody()); - return null; - } - return ingressHostList; - } - - /** - * List all of the ingresses in the specified namespace. - * - * @param namespace the namespace to which the ingresses belong - * @return a list of ingress names in the namespace - * @throws ApiException if Kubernetes client API call fails - */ - public static List listIngresses(String namespace) throws ApiException { - - List ingressNames = new ArrayList<>(); - NetworkingV1beta1IngressList ingressList = Kubernetes.listNamespacedIngresses(namespace); - List listOfIngress = ingressList.getItems(); - - listOfIngress.forEach(ingress -> { - if (ingress.getMetadata() != null) { - ingressNames.add(ingress.getMetadata().getName()); - } - }); - - return ingressNames; - } } diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/ApacheParams.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/ApacheParams.java index 155055775ea..b94ce74bae7 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/ApacheParams.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/ApacheParams.java @@ -10,11 +10,10 @@ import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams; /** - * All parameters needed to install Apache ingress controller. + * All parameters needed to install Apache load balancer. */ public class ApacheParams { - // Only add the values which need to be updated here. // The default values can be found here: // weblogic-kubernetes-operator/kubernetes/samples/charts/apache-webtier/values.yaml private static final String IMAGE = "image"; diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java index 3637799e6a7..5f548147f94 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java @@ -10,7 +10,6 @@ import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.models.V1Secret; import oracle.weblogic.kubernetes.actions.impl.LoggingExporter; -import oracle.weblogic.kubernetes.actions.impl.Secret; import oracle.weblogic.kubernetes.assertions.impl.Apache; import oracle.weblogic.kubernetes.assertions.impl.Application; import oracle.weblogic.kubernetes.assertions.impl.ClusterRole; @@ -33,6 +32,8 @@ import oracle.weblogic.kubernetes.assertions.impl.WitAssertion; import org.joda.time.DateTime; +import static oracle.weblogic.kubernetes.actions.TestActions.listSecrets; + /** * General assertions needed by the tests to validate CRD, Domain, Pods etc. */ @@ -71,7 +72,7 @@ public static Callable isNginxReady(String namespace) { /** * Check if there are ready Apache pods in the specified namespace. * - * @param namespace in which to check if APache pods are in the ready state + * @param namespace in which to check if Apache pods are in the ready state * @return true if there are ready Apache pods in the specified namespace , false otherwise */ public static Callable isApacheReady(String namespace) { @@ -631,7 +632,7 @@ public static boolean clusterRoleBindingExists(String clusterRoleBindingName) th * @return true if secret exists, false otherwise */ public static boolean secretExists(String secretName, String namespace) { - for (V1Secret secret : Secret.listSecrets(namespace).getItems()) { + for (V1Secret secret : listSecrets(namespace).getItems()) { if (secret.getMetadata() != null) { String name = secret.getMetadata().getName(); if (name != null && name.equals(secretName)) { diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Apache.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Apache.java index ab2b5522964..bd9aeb103be 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Apache.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Apache.java @@ -6,7 +6,7 @@ import java.util.concurrent.Callable; /** - * Assertions for Apache ingress controller. + * Assertions for Apache load balancer. */ public class Apache { diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Kubernetes.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Kubernetes.java index b15c27caa18..a4cac8c8223 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Kubernetes.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Kubernetes.java @@ -347,7 +347,7 @@ public static boolean isNginxPodReady(String namespace) throws ApiException { /** * Checks if a Apache pod is running in the specified namespace. - * The method assumes that the Apache pod name contains "APACHE_RELEAE_NAME-apache-webtier". + * The method assumes the Apache pod name contains "APACHE_RELEAE_NAME-namespace.substring(3)-apache-webtier". * * @param namespace in which to check if the Apache pod is running * @return true if the pod is running, otherwise false @@ -361,7 +361,7 @@ public static boolean isApachePodRunning(String namespace) throws ApiException { /** * Check whether the Apache pod is ready in the specified namespace. - * The method assumes that the Apache pod name starts with "APACHE_RELEASE_NAME_apache-webtier". + * The method assumes the Apache pod name contains "APACHE_RELEASE_NAME-namespace.substring(3)-apache-webtier". * * @param namespace in which to check if the Apache pod is ready * @return true if the pod is in the ready state, false otherwise diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java index b7f395d9bc0..6d38a398221 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java @@ -640,7 +640,7 @@ public static HelmParams installAndVerifyApache(String apacheNamespace, * @param httpsNodePort the https nodeport of Apache * @param domainUid the uid of the domain to which Apache will route the services * @param volumePath the path to put your own custom_mod_wl_apache.conf file - * @param virtualHostName the VirtualHostName of the Apache HTTP server which is used to enables custom SSL config + * @param virtualHostName the VirtualHostName of the Apache HTTP server which is used to enable custom SSL config * @param clusterNamePortMap the map with clusterName as key and cluster port number as value * @return the Apache Helm installation parameters */ @@ -656,7 +656,7 @@ public static HelmParams installAndVerifyApache(String apacheNamespace, LoggingFacade logger = getLogger(); - // Create Docker registry secret in the operator namespace to pull the image from repository + // Create Docker registry secret in the apache namespace to pull the Apache webtier image from repository if (!secretExists(REPO_SECRET_NAME, apacheNamespace)) { logger.info("Creating Docker registry secret in namespace {0}", apacheNamespace); createDockerRegistrySecret(apacheNamespace); @@ -758,12 +758,12 @@ public static HelmParams installAndVerifyApache(String apacheNamespace, // verify that Apache is installed logger.info("Checking Apache release {0} status in namespace {1}", - APACHE_RELEASE_NAME, apacheNamespace); - assertTrue(isHelmReleaseDeployed(APACHE_RELEASE_NAME, apacheNamespace), + APACHE_RELEASE_NAME + "-" + apacheNamespace.substring(3), apacheNamespace); + assertTrue(isHelmReleaseDeployed(APACHE_RELEASE_NAME + "-" + apacheNamespace.substring(3), apacheNamespace), String.format("Apache release %s is not in deployed status in namespace %s", - APACHE_RELEASE_NAME, apacheNamespace)); + APACHE_RELEASE_NAME + "-" + apacheNamespace.substring(3), apacheNamespace)); logger.info("Apache release {0} status is deployed in namespace {1}", - APACHE_RELEASE_NAME, apacheNamespace); + APACHE_RELEASE_NAME + "-" + apacheNamespace.substring(3), apacheNamespace); // wait until the Apache pod is ready. withStandardRetryPolicy From 9ad4a3b108e424b2af503db7540e071621dca7c2 Mon Sep 17 00:00:00 2001 From: vanajamukkara Date: Mon, 31 Aug 2020 18:36:05 +0000 Subject: [PATCH 12/15] move OCR login to test --- .../kubernetes/extensions/ImageBuilders.java | 73 ++++++++++--------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/ImageBuilders.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/ImageBuilders.java index a8745cd42b1..ff485d153f2 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/ImageBuilders.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/ImageBuilders.java @@ -135,6 +135,44 @@ public void beforeAll(ExtensionContext context) { assertFalse(operatorImage.isEmpty(), "Image name can not be empty"); assertTrue(Operator.buildImage(operatorImage), "docker build failed for Operator"); + // docker login to OCR if OCR_USERNAME and OCR_PASSWORD is provided in env var + if (!OCR_USERNAME.equals(REPO_DUMMY_VALUE)) { + withStandardRetryPolicy + .conditionEvaluationListener( + condition -> logger.info("Waiting for docker login to be successful" + + "(elapsed time {0} ms, remaining time {1} ms)", + condition.getElapsedTimeInMS(), + condition.getRemainingTimeInMS())) + .until(() -> dockerLogin(OCR_REGISTRY, OCR_USERNAME, OCR_PASSWORD)); + } + + // The following code is for pulling WLS images if running tests in Kind cluster + if (KIND_REPO != null) { + // We can't figure out why the kind clusters can't pull images from OCR using the image pull secret. There + // is some evidence it may be a containerd bug. Therefore, we are going to "give up" and workaround the issue. + // The workaround will be to: + // 1. docker login + // 2. docker pull + // 3. docker tag with the KIND_REPO value + // 4. docker push this new image name + // 5. use this image name to create the domain resource + Collection images = new ArrayList<>(); + images.add(WLS_BASE_IMAGE_NAME + ":" + WLS_BASE_IMAGE_TAG); + images.add(JRF_BASE_IMAGE_NAME + ":" + JRF_BASE_IMAGE_TAG); + images.add(DB_IMAGE_NAME + ":" + DB_IMAGE_TAG); + + for (String image : images) { + withStandardRetryPolicy + .conditionEvaluationListener( + condition -> logger.info("Waiting for pullImageFromOcrAndPushToKind for image {0} to be successful" + + "(elapsed time {1} ms, remaining time {2} ms)", image, + condition.getElapsedTimeInMS(), + condition.getRemainingTimeInMS())) + .until(pullImageFromOcrAndPushToKind(image) + ); + } + } + if (System.getenv("SKIP_BASIC_IMAGE_BUILD") == null) { // build MII basic image miiBasicImage = MII_BASIC_IMAGE_NAME + ":" + MII_BASIC_IMAGE_TAG; @@ -208,41 +246,6 @@ public void beforeAll(ExtensionContext context) { } } - // The following code is for pulling WLS images if running tests in Kind cluster - if (KIND_REPO != null) { - // We can't figure out why the kind clusters can't pull images from OCR using the image pull secret. There - // is some evidence it may be a containerd bug. Therefore, we are going to "give up" and workaround the issue. - // The workaround will be to: - // 1. docker login - // 2. docker pull - // 3. docker tag with the KIND_REPO value - // 4. docker push this new image name - // 5. use this image name to create the domain resource - Collection images = new ArrayList<>(); - images.add(WLS_BASE_IMAGE_NAME + ":" + WLS_BASE_IMAGE_TAG); - images.add(JRF_BASE_IMAGE_NAME + ":" + JRF_BASE_IMAGE_TAG); - images.add(DB_IMAGE_NAME + ":" + DB_IMAGE_TAG); - - withStandardRetryPolicy - .conditionEvaluationListener( - condition -> logger.info("Waiting for docker login to be successful" - + "(elapsed time {0} ms, remaining time {1} ms)", - condition.getElapsedTimeInMS(), - condition.getRemainingTimeInMS())) - .until(() -> dockerLogin(OCR_REGISTRY, OCR_USERNAME, OCR_PASSWORD)); - - for (String image : images) { - withStandardRetryPolicy - .conditionEvaluationListener( - condition -> logger.info("Waiting for pullImageFromOcrAndPushToKind for image {0} to be successful" - + "(elapsed time {1} ms, remaining time {2} ms)", image, - condition.getElapsedTimeInMS(), - condition.getRemainingTimeInMS())) - .until(pullImageFromOcrAndPushToKind(image) - ); - } - } - // set initialization success to true, not counting the istio installation as not all tests use istio isInitializationSuccessful = true; logger.info("Installing istio before any test suites are run"); From 39e5e8fb9a54cf744d8b255d3dbaf05e3d3375a8 Mon Sep 17 00:00:00 2001 From: xiancao Date: Mon, 31 Aug 2020 23:21:19 +0000 Subject: [PATCH 13/15] address Vanaja's comments --- .../kubernetes/ItTwoDomainsLoadBalancers.java | 19 ++++++++++++++----- .../weblogic/kubernetes/TestConstants.java | 4 +++- .../kubernetes/utils/CommonTestUtils.java | 10 ++++++++++ 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java index ce3d734575c..b6b1d43e800 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java @@ -84,7 +84,7 @@ import static oracle.weblogic.kubernetes.TestConstants.ADMIN_PASSWORD_DEFAULT; import static oracle.weblogic.kubernetes.TestConstants.ADMIN_SERVER_NAME_BASE; import static oracle.weblogic.kubernetes.TestConstants.ADMIN_USERNAME_DEFAULT; -import static oracle.weblogic.kubernetes.TestConstants.APACHE_IMAGE_12213; +import static oracle.weblogic.kubernetes.TestConstants.APACHE_IMAGE; import static oracle.weblogic.kubernetes.TestConstants.APACHE_RELEASE_NAME; import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_API_VERSION; import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_VERSION; @@ -191,7 +191,7 @@ public class ItTwoDomainsLoadBalancers { private static Path dstFile = null; private static HelmParams apacheHelmParams1 = null; private static HelmParams apacheHelmParams2 = null; - private static String kindRepoApacheImage = APACHE_IMAGE_12213; + private static String kindRepoApacheImage = APACHE_IMAGE; // domain constants private final String clusterName = "cluster-1"; @@ -280,10 +280,10 @@ public static void initAll(@Namespaces(6) List namespaces) { withStandardRetryPolicy .conditionEvaluationListener( condition -> logger.info("Waiting for pullImageFromOcirAndPushToKind for image {0} to be successful" - + "(elapsed time {1} ms, remaining time {2} ms)", APACHE_IMAGE_12213, + + "(elapsed time {1} ms, remaining time {2} ms)", APACHE_IMAGE, condition.getElapsedTimeInMS(), condition.getRemainingTimeInMS())) - .until(pullImageFromOcirAndPushToKind(APACHE_IMAGE_12213)); + .until(pullImageFromOcirAndPushToKind(APACHE_IMAGE)); } } @@ -476,6 +476,10 @@ public void testVoyagerHostHttpRoutingAcrossDomains() { /** * Verify Apache load balancer default sample through HTTP channel. + * Configure the Apache webtier as a load balancer for a WebLogic domain using the default configuration. + * It only support HTTP protocol. + * For details, please see + * https://github.com/oracle/weblogic-kubernetes-operator/tree/master/kubernetes/samples/charts/apache-samples/default-sample */ @Order(8) @Test @@ -490,6 +494,11 @@ public void testApacheDefaultSample() { /** * Verify Apache load balancer custom sample through HTTP and HTTPS channel. + * Configure the Apache webtier as a load balancer for multiple WebLogic domains using a custom configuration. + * Create a custom Apache plugin configuration file named custom_mod_wl_apache.conf in a directory specified + * in helm chart parameter volumePath. + * For more details, please check: + * https://github.com/oracle/weblogic-kubernetes-operator/tree/master/kubernetes/samples/charts/apache-samples/custom-sample */ @Order(9) @Test @@ -1448,7 +1457,7 @@ private void verifyClusterLoadbalancing(String domainUid, for (int i = 0; i < 10; i++) { ExecResult result; try { - logger.info(curlRequest); + logger.info("executing curl command: {0}", curlRequest); result = ExecCommand.exec(curlRequest, true); String response = result.stdout().trim(); logger.info("Response for iteration {0}: exitValue {1}, stdout {2}, stderr {3}", diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java index 4b97420f710..88784a6a548 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java @@ -94,7 +94,9 @@ public interface TestConstants { public static final String VOYAGER_CHART_VERSION = "12.0.0"; // Apache constants - public static final String APACHE_IMAGE_12213 = "phx.ocir.io/weblogick8s/oracle/apache:12.2.1.3"; + public static final String APACHE_IMAGE_NAME = "phx.ocir.io/weblogick8s/oracle/apache"; + public static final String APACHE_IMAGE_VERSION = "12.2.1.3"; + public static final String APACHE_IMAGE = APACHE_IMAGE_NAME + ":" + APACHE_IMAGE_VERSION; public static final String APACHE_RELEASE_NAME = "apache-release" + BUILD_ID; public static final String APACHE_SAMPLE_CHART_DIR = "../kubernetes/samples/charts/apache-webtier"; diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java index 6d38a398221..866a718aa66 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java @@ -688,6 +688,8 @@ public static HelmParams installAndVerifyApache(String apacheNamespace, if (volumePath != null && clusterNamePortMap != null) { // create a custom Apache plugin configuration file named custom_mod_wl_apache.conf + // and put it under the directory specified in volumePath + // this file provides a custom Apache plugin configuration to fine tune the behavior of Apache Path customConf = Paths.get(volumePath, "custom_mod_wl_apache.conf"); ArrayList lines = new ArrayList<>(); lines.add(""); @@ -695,12 +697,20 @@ public static HelmParams installAndVerifyApache(String apacheNamespace, lines.add("WebLogicPort ${WEBLOGIC_PORT}"); lines.add(""); + // Directive for weblogic admin Console deployed on Weblogic Admin Server lines.add(""); lines.add("SetHandler weblogic-handler"); lines.add("WebLogicHost " + domainUid + "-admin-server"); lines.add("WebLogicPort ${WEBLOGIC_PORT}"); lines.add(""); + // Directive for all application deployed on weblogic cluster with a prepath defined by LOCATION variable + // For example, if the LOCAITON is set to '/weblogic1', all applications deployed on the cluster can be accessed + // via http://myhost:myport/weblogic1/application_end_url + // where 'myhost' is the IP of the machine that runs the Apache web tier, and + // 'myport' is the port that the Apache web tier is publicly exposed to. + // Note that LOCATION cannot be set to '/' unless this is the only Location module configured. + // create a LOCATION variable for each domain cluster int i = 1; for (String clusterName : clusterNamePortMap.keySet()) { lines.add(""); From 69c7c22d25b7481ee49227db9d22e39727a784f0 Mon Sep 17 00:00:00 2001 From: xiancao Date: Tue, 1 Sep 2020 19:10:18 +0000 Subject: [PATCH 14/15] address Pani's comments --- .../oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java | 4 ++-- .../oracle/weblogic/kubernetes/extensions/ImageBuilders.java | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java index b6b1d43e800..c9ad42bff80 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItTwoDomainsLoadBalancers.java @@ -257,8 +257,8 @@ public static void initAll(@Namespaces(6) List namespaces) { image = kindRepoImage; isUseSecret = false; - // We do not know why the kind clusters can't pull Apache webtier image from OCIR using the image pull secret. - // Try the following instead + // The kind clusters can't pull Apache webtier image from OCIR using the image pull secret. + // Try the following instead: // 1. docker login // 2. docker pull // 3. docker tag with the KIND_REPO value diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/ImageBuilders.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/ImageBuilders.java index ff485d153f2..7fd7511afcf 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/ImageBuilders.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/ImageBuilders.java @@ -148,8 +148,7 @@ public void beforeAll(ExtensionContext context) { // The following code is for pulling WLS images if running tests in Kind cluster if (KIND_REPO != null) { - // We can't figure out why the kind clusters can't pull images from OCR using the image pull secret. There - // is some evidence it may be a containerd bug. Therefore, we are going to "give up" and workaround the issue. + // The kind clusters can't pull images from OCR using the image pull secret. // The workaround will be to: // 1. docker login // 2. docker pull From 485b2fa886ae809e7e8c43714697f2d941d900d2 Mon Sep 17 00:00:00 2001 From: xiancao Date: Tue, 1 Sep 2020 19:32:47 +0000 Subject: [PATCH 15/15] address Vanaja's comments --- .../oracle/weblogic/kubernetes/extensions/ImageBuilders.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/ImageBuilders.java b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/ImageBuilders.java index 7fd7511afcf..0575ec93168 100644 --- a/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/ImageBuilders.java +++ b/new-integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/ImageBuilders.java @@ -148,7 +148,8 @@ public void beforeAll(ExtensionContext context) { // The following code is for pulling WLS images if running tests in Kind cluster if (KIND_REPO != null) { - // The kind clusters can't pull images from OCR using the image pull secret. + // The kind clusters can't pull images from OCR using the image pull secret. + // It may be a containerd bug. We are going to workaround this issue. // The workaround will be to: // 1. docker login // 2. docker pull