Backport (release/3.4) resolution to intermittent WlsSample tests failures due to NGINX download issue

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItWlsSamples.java

@@ -10,7 +10,9 @@
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Stream;
 
 import oracle.weblogic.kubernetes.actions.impl.primitive.Command;
@@ -38,11 +40,16 @@
 import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_VERSION;
 import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST;
 import static oracle.weblogic.kubernetes.TestConstants.KIND_REPO;
+import static oracle.weblogic.kubernetes.TestConstants.NGINX_INGRESS_IMAGE_DIGEST;
+import static oracle.weblogic.kubernetes.TestConstants.OCIR_NGINX_IMAGE_NAME;
+import static oracle.weblogic.kubernetes.TestConstants.OCIR_REGISTRY;
+import static oracle.weblogic.kubernetes.TestConstants.OCIR_SECRET_NAME;
 import static oracle.weblogic.kubernetes.TestConstants.OCIR_WEBLOGIC_IMAGE_TAG;
 import static oracle.weblogic.kubernetes.TestConstants.PV_ROOT;
 import static oracle.weblogic.kubernetes.TestConstants.WEBLOGIC_IMAGE_TO_USE_IN_SPEC;
 import static oracle.weblogic.kubernetes.actions.ActionConstants.ITTESTS_DIR;
 import static oracle.weblogic.kubernetes.actions.ActionConstants.MODEL_DIR;
+import static oracle.weblogic.kubernetes.actions.ActionConstants.RESOURCE_DIR;
 import static oracle.weblogic.kubernetes.actions.ActionConstants.WORK_DIR;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainDoesNotExist;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainExists;
@@ -53,6 +60,7 @@
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getDateAndTimeStamp;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntil;
+import static oracle.weblogic.kubernetes.utils.FileUtils.generateFileFromTemplate;
 import static oracle.weblogic.kubernetes.utils.FileUtils.replaceStringInFile;
 import static oracle.weblogic.kubernetes.utils.ImageUtils.createOcirRepoSecret;
 import static oracle.weblogic.kubernetes.utils.ImageUtils.createSecretForBaseImages;
@@ -140,6 +148,7 @@ public static void initAll(@Namespaces(4) List<String> namespaces) {
     logger.info("Assign a unique namespace for Nginx controller");
     assertNotNull(namespaces.get(3), "Namespace is null");
     nginxNamespace = namespaces.get(3);
+    createOcirRepoSecret(nginxNamespace);
 
     // create pull secrets for WebLogic image when running in non Kind Kubernetes cluster
     // this secret is used only for non-kind cluster
@@ -376,9 +385,19 @@ void testTraefikIngressController() {
   @DisplayName("Manage Nginx Ingress Controller with setupLoadBalancer")
   void testNginxIngressController() {
     setupSample();
+    Map<String, String> templateMap  = new HashMap<>();
+    templateMap.put("TEST_IMAGES_REPO", OCIR_REGISTRY);
+    templateMap.put("TEST_NGINX_IMAGE_NAME ",  OCIR_NGINX_IMAGE_NAME);
+    templateMap.put("NGINX_INGRESS_IMAGE_DIGEST",NGINX_INGRESS_IMAGE_DIGEST);
+    templateMap.put("TEST_IMAGES_REPO_SECRET_NAME", OCIR_SECRET_NAME);
+    Path srcPropFile = Paths.get(RESOURCE_DIR, "nginx.template.properties");
+    Path targetPropFile = assertDoesNotThrow(
+        () -> generateFileFromTemplate(srcPropFile.toString(), "nginx.properties", templateMap));
+    logger.info("Generated nginx.properties file path is {0}", targetPropFile);
     Path scriptBase = Paths.get(tempSamplePath.toString(), "charts/util");
-    setupLoadBalancer(scriptBase, "nginx", " -c -n " + nginxNamespace);
-    setupLoadBalancer(scriptBase, "nginx", " -d -n " + nginxNamespace);
+    setupLoadBalancer(scriptBase, "nginx", " -c -n " + nginxNamespace
+         + " -p " + targetPropFile.toString());
+    setupLoadBalancer(scriptBase, "nginx", " -d -s -n " + nginxNamespace);
   }
 
   // Function to execute domain lifecyle scripts

integration-tests/src/test/resources/bash-scripts/images.properties

@@ -58,3 +58,7 @@ k8s.gcr.io/ingress-nginx/controller:v1.2.0;weblogick8s/test-images/ingress-nginx
 
 # Oracle DB Operator Image
 container-registry.oracle.com/database/operator:0.1.0;weblogick8s/test-images/database/operator:0.1.0
+
+# ElasticSearch Image
+elasticsearch:7.8.1;weblogick8s/test-images/docker/elasticsearch:7.8.1
+

integration-tests/src/test/resources/nginx.template.properties

@@ -0,0 +1 @@
+--set controller.image.registry=TEST_IMAGES_REPO --set controller.image.image=TEST_NGINX_IMAGE_NAME   --set controller.image.digest=NGINX_INGRESS_IMAGE_DIGEST --set imagePullSecrets[0].name=TEST_IMAGES_REPO_SECRET_NAME

kubernetes/samples/charts/util/setupLoadBalancer.sh

@@ -2,16 +2,16 @@
 # Copyright (c) 2018, 2022, Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
-# This script create or delete an Ingress controller.
+# This script create or delete an Ingress controller. 
 #  The script supports ingress controllers: Traefik and Nginx.
 
 set -eu
 set -o pipefail
 
 UTILDIR="$(dirname "$(readlink -f "$0")")"
 
-#Kubernetes command line interface.
-#Default is 'kubectl' if KUBERNETES_CLI env variable is not set.
+#Kubernetes command line interface. 
+#Default is 'kubectl' if KUBERNETES_CLI env variable is not set.  
 kubernetesCli=${KUBERNETES_CLI:-kubectl}
 
 # https://github.com/containous/traefik/releases
@@ -28,6 +28,8 @@ namespace=""
 release=""
 repository=""
 chart=""
+ingressPropFile="ingress.properties"
+skipDeleteNamespace="false"
 
 # timestamp
 #   purpose:  echo timestamp in the form yyyy-mm-ddThh:mm:ss.nnnnnnZ
@@ -60,6 +62,8 @@ usage() {
     -t <ingress type>    : ingress type traefik or nginx [required]
     -v <ingress version> : ingress release version
     -n <namespace>       : ingress namespace
+    -p <ingress-prop>    : extra ingress helm properties 
+    -s                   : skip deleting ingress namespace
     -m <kubernetes_cli>  : Kubernetes command line interface. Default is 'kubectl' if KUBERNETES_CLI env variable is not set. Otherwise default is the value of KUBERNETES_CLI env variable.
     -h                   : print help
 EOF
@@ -68,26 +72,37 @@ exit $1
 
 action_chosen=false
 
-while getopts "cdt:n:r:v:h" opt; do
+while getopts "scdt:p:n:r:v:h" opt; do
   case $opt in
     c) action="create"
        if [ $action_chosen = "true" ]; then
         printError " Both -c (create) and -d (delete) option can not be specified for ingress controller."
         usage 1
-       fi
+       fi 
        action_chosen=true
     ;;
     d) action="delete"
        if [ $action_chosen = "true" ]; then
         printError " Both -c (create) and -d (delete) option can not be specified for ingress controller."
         usage 1
-       fi
+       fi 
        action_chosen=true
     ;;
+    s) skipDeleteNamespace="true"
+       printInfo "Will Skip the Namespace Deletion"
+    ;;
     n) namespace="${OPTARG}"
     ;;
     t) ingressType="${OPTARG}"
     ;;
+    p) ingressPropFile="${OPTARG}"
+       if [ ${action} == "create" ]; then
+         if [ ! -f ${ingressPropFile} ]; then
+          printError "[create] action is choosen but the custom ingress property file [${ingressPropFile}] is missing."
+          usage 1
+         fi
+      fi 
+    ;;
     v) release="${OPTARG}"
     ;;
     m) kubernetesCli="${OPTARG}"
@@ -100,23 +115,23 @@ while getopts "cdt:n:r:v:h" opt; do
 done
 
 if [ "x${action}" == "x" ]; then
- printError "You must specify either -c (create) or -d (delete) ingress controller"
+ printError "You must specify either -c (create) or -d (delete) ingress controller" 
  usage 1
 fi
 
 if [ "x${ingressType}" == "x" ]; then
  printError "You must specify ingress type (traefik or nginx) thru -t option"
  usage 1
-fi
+fi 
 
-case  ${ingressType} in
-   "traefik")
+case  ${ingressType} in 
+   "traefik") 
               [[ -z "${release}"   ]] && release="${DefaultTraefikVersion}"
               [[ -z "${namespace}" ]] && namespace="${ingressType}"
               repository="traefik"
               chart="traefik-release"
               ;;
-    "nginx")
+    "nginx")   
               [[ -z "${release}"   ]] && release="${DefaultNginxVersion}"
               [[ -z "${namespace}" ]] && namespace="${ingressType}"
               repository="ingress-nginx"
@@ -163,12 +178,12 @@ waitForIngressPod() {
   if [ $? != 0 ]; then
    printError "${type} ingress controller pod not READY in state in 5 min"
    exit -1;
-  else
+  else 
    ipod=$(${kubernetesCli} get pod -n ${ns} -l app.kubernetes.io/instance=${type}-release -o jsonpath="{.items[0].metadata.name}")
    ${kubernetesCli} get po/${ipod} -n ${ns}
    helm list -n ${ns}
-  fi
- }
+  fi 
+ } 
 
 createTraefik() {
   ns=${1}
@@ -184,13 +199,15 @@ createTraefik() {
     printInfo "Traefik chart repository is already added."
   fi
 
+  # load the extra set of helm values if provided thru file using -p option
   if [ "$(helm list -q -n ${ns} | grep $chart | wc -l)" = 0 ]; then
     printInfo "Installing Traefik controller on namespace ${ns}"
     purgeDefaultResources || true
     helm install $chart traefik/traefik --namespace ${ns} \
+     $(cat ${ingressPropFile} 2>&- || false ) \
      --set image.tag=${rel} \
-     --values ${UTILDIR}/../traefik/values.yaml
-    if [ $? != 0 ]; then
+     --values ${UTILDIR}/../traefik/values.yaml 
+    if [ $? != 0 ]; then 
      printError "Helm installation of the Traefik ingress controller failed."
      exit -1;
     fi
@@ -202,46 +219,49 @@ createTraefik() {
   tpod=$(${kubernetesCli} -o name get po -n ${ns})
   traefik_image=$(${kubernetesCli} get ${tpod} -n ${ns} -o jsonpath='{.spec.containers[0].image}')
   printInfo "Traefik image chosen [${traefik_image}]"
+  helm get values $chart --namespace ${ns} 
 }
 
 # Remove ingress related resources from default Namespace ( if any )
 purgeDefaultResources() {
    printInfo "Remove ingress related resources from default Namespace (if any)"
   croles=$(${kubernetesCli} get ClusterRole | grep ${chart} | awk '{print $1}')
-  for crole in ${croles}; do
+  for crole in ${croles}; do 
    printInfo "Deleting ClusterRole ${crole} from default Namespace"
-   ${kubernetesCli} delete ClusterRole ${crole}
+   ${kubernetesCli} delete ClusterRole ${crole} 
   done
 
   crbs=$(${kubernetesCli} get ClusterRoleBinding | grep ${chart} | awk '{print $1}')
-  for crb in ${crbs}; do
+  for crb in ${crbs}; do 
    printInfo "Deleting ClusterRoleBinding ${crb} from default Namespace"
-   ${kubernetesCli} delete ClusterRoleBinding ${crb}
+   ${kubernetesCli} delete ClusterRoleBinding ${crb} 
   done
 
   vwcs=$(${kubernetesCli} get ValidatingWebhookConfiguration | grep ${chart} | awk '{print $1}')
-  for vwc in ${vwcs}; do
+  for vwc in ${vwcs}; do 
     printInfo "Deleting ValidatingWebhookConfiguration ${vwc} from default Namespace"
-    ${kubernetesCli} delete ValidatingWebhookConfiguration ${vwc}
+    ${kubernetesCli} delete ValidatingWebhookConfiguration ${vwc} 
   done
 }
 
 deleteIngress() {
   type=${1}
   ns=${2}
   if [ "$(helm list --namespace $ns | grep $chart |  wc -l)" = 1 ]; then
-    printInfo "Deleting ${type} controller from namespace $ns"
+    printInfo "Deleting ${type} controller from namespace $ns" 
     helm uninstall --namespace $ns $chart
     ${kubernetesCli} wait --namespace ${ns} \
        --for=delete pod \
        --selector=app.kubernetes.io/instance=${type}-release \
        --timeout=120s
-    ${kubernetesCli} delete ns ${ns}
-    ${kubernetesCli} wait --for=delete namespace ${ns} --timeout=60s || true
+    if [ ${skipDeleteNamespace} == "false" ]; then
+      ${kubernetesCli} delete ns ${ns}
+      ${kubernetesCli} wait --for=delete namespace ${ns} --timeout=60s || true
+    fi
     printInfo "Remove ${type} chart repository [${repository}] "
     helm repo remove ${repository}
   else
-    printInfo "${type} controller has already been deleted from namespace [${ns}] or not installed in the namespace [${ns}]."
+    printInfo "${type} controller has already been deleted from namespace [${ns}] or not installed in the namespace [${ns}]." 
   fi
 
   if [ "${ingressType}" = traefik ]; then
@@ -256,7 +276,7 @@ createNginx() {
   release=${2}
   chart="nginx-release"
   createNameSpace $ns || true
-  printInfo "Creating Nginx controller on namespace ${ns}"
+  printInfo "Creating Nginx controller on namespace ${ns}" 
 
   if [ "$(helm search repo ingress-nginx | grep nginx | wc -l)" = 0 ]; then
     printInfo "Add Nginx chart repository"
@@ -265,11 +285,15 @@ createNginx() {
     printInfo "Nginx chart repository is already added."
   fi
 
+  # load the extra set of helm values if provided thru file using -p option
   if [ "$(helm list --namespace ${ns} | grep $chart |  wc -l)" = 0 ]; then
     purgeDefaultResources || true
     helm install $chart ingress-nginx/ingress-nginx \
-         --set "controller.admissionWebhooks.enabled=false" \
-         --namespace ${ns} --version ${release}
+      $(cat ${ingressPropFile} 2>&- || false ) \
+      --set "controller.admissionWebhooks.enabled=false" \
+      --namespace ${ns} --version ${release} \
+      --set "controller.image.pullPolicy=IfNotPresent" \
+      --set "controller.image.tag=v1.2.0" 
     if [ $? != 0 ]; then
      printError "Helm installation of the Nginx ingress controller failed."
      exit -1;
@@ -282,6 +306,7 @@ createNginx() {
   waitForIngressPod nginx ${ns}
   tpod=$(${kubernetesCli} -o name get po -n ${ns})
   ${kubernetesCli} describe ${tpod} -n ${ns}
+  helm get values $chart -n ${ns}
 }
 
 main() {
@@ -303,4 +328,4 @@ main() {
   fi
 }
 
-main "$@"
+main "$@"