Integration Test: Add error path testing for Auxiliary Image feature

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItManageNameSpace.java

@@ -41,7 +41,6 @@
 import static oracle.weblogic.kubernetes.TestConstants.ADMIN_SERVER_NAME_BASE;
 import static oracle.weblogic.kubernetes.TestConstants.ADMIN_USERNAME_DEFAULT;
 import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_API_VERSION;
-import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_VERSION;
 import static oracle.weblogic.kubernetes.TestConstants.MANAGED_SERVER_NAME_BASE;
 import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_NAME;
 import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_TAG;
@@ -58,12 +57,12 @@
 import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort;
 import static oracle.weblogic.kubernetes.actions.TestActions.scaleClusterWithRestApi;
 import static oracle.weblogic.kubernetes.actions.TestActions.uninstallOperator;
-import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainDoesNotExist;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodDoesNotExist;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDomainAndVerify;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createOcirRepoSecret;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createSecretWithUsernamePassword;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.deleteDomainResource;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyOperator;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.scaleAndVerifyCluster;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.setPodAntiAffinity;
@@ -382,23 +381,6 @@ private void checkUpgradeFailedToAddNSManagedByAnotherOperator() {
         + " managed by other operator");
   }
 
-  private void deleteDomainResource(String domainNS, String domainUid) {
-    //clean up domain resources in namespace and set namespace to label , managed by operator
-    logger.info("deleting domain custom resource {0}", domainUid);
-    assertTrue(deleteDomainCustomResource(domainUid, domainNS));
-
-    // wait until domain was deleted
-    withStandardRetryPolicy
-        .conditionEvaluationListener(
-            condition -> logger.info("Waiting for domain {0} to be deleted in namespace {1} "
-                    + "(elapsed time {2}ms, remaining time {3}ms)",
-                domainUid,
-                domainNS,
-                condition.getElapsedTimeInMS(),
-                condition.getRemainingTimeInMS()))
-        .until(domainDoesNotExist(domainUid, DOMAIN_VERSION, domainNS));
-  }
-
   private HelmParams installAndVerifyOperatorCanManageDomainBySelector(Map<String,String> managedDomains,
                                                                        Map<String,String> unmanagedDomains,
                                                                        String selector, String selectorValue,

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItParameterizedDomain.java

@@ -119,6 +119,7 @@
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.clusterRoleBindingExists;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.clusterRoleExists;
 import static oracle.weblogic.kubernetes.utils.CommonPatchTestUtils.patchDomainResource;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodLogContainsString;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReady;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDomainAndVerify;
@@ -306,7 +307,7 @@ public void testOperatorLogSevereMsg() {
     createMiiDomainNegative("miidomainnegative", miiDomainNegativeNamespace);
     String operatorPodName =
         assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace));
-    waitForPodLogContainsString(opNamespace, operatorPodName,
+    checkPodLogContainsString(opNamespace, operatorPodName,
         "Domain miidomainnegative is not valid: RuntimeEncryption secret '" + encryptionSecretName
         + "' not found in namespace '" + miiDomainNegativeNamespace + "'");
   }
@@ -1545,31 +1546,6 @@ private void waitForFileExistsInPod(String namespace, String podName, String fil
             "fileExistsInPod failed with IOException, ApiException or InterruptedException"));
   }
 
-  private Callable<Boolean> podLogContainsString(String namespace, String podName, String expectedString) {
-    return () -> {
-      logger.info("pod name: {0}", podName);
-      String podLog = getPodLog(podName, namespace);
-      logger.info("pod log: {0}", podLog);
-      return podLog.contains(expectedString);
-    };
-  }
-
-  private void waitForPodLogContainsString(String namespace, String podName, String expectedString) {
-
-    logger.info("Wait for string {0} existing in pod {1} in namespace {2}", expectedString, podName, namespace);
-    withStandardRetryPolicy
-        .conditionEvaluationListener(
-            condition -> logger.info("Waiting for string {0} existing in pod {1} in namespace {2} "
-                    + "(elapsed time {3}ms, remaining time {4}ms)",
-                expectedString,
-                podName,
-                namespace,
-                condition.getElapsedTimeInMS(),
-                condition.getRemainingTimeInMS()))
-        .until(assertDoesNotThrow(() -> podLogContainsString(namespace, podName, expectedString),
-            "podLogContainsString failed with IOException, ApiException or InterruptedException"));
-  }
-
   /**
    * Negative test case for creating a model-in-image domain without encryption secret created.
    * The admin server service/pod will not be created.

integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java

@@ -299,6 +299,51 @@ public static Domain createDomainResource(
     return domainCR;
   }
 
+  /**
+   * Create a domain object for a Kubernetes domain custom resource using the basic WLS image and auxiliary image.
+   *
+   * @param domainResourceName name of the domain resource
+   * @param domNamespace Kubernetes namespace that the domain is hosted
+   * @param baseImageName name of the base image to use
+   * @param adminSecretName name of the new WebLogic admin credentials secret
+   * @param repoSecretName name of the secret for pulling the WebLogic image
+   * @param encryptionSecretName name of the secret used to encrypt the models
+   * @param replicaCount number of managed servers to start
+   * @param clusterName name of the cluster to add in domain
+   * @param auxiliaryImageVolumes list of AuxiliaryImageVolumes
+   * @param auxiliaryImages list of AuxiliaryImages
+   * @return domain object of the domain resource
+   */
+  public static Domain createDomainResourceWithAuxiliaryImage(
+      String domainResourceName,
+      String domNamespace,
+      String baseImageName,
+      String adminSecretName,
+      String repoSecretName,
+      String encryptionSecretName,
+      int replicaCount,
+      String clusterName,
+      List<AuxiliaryImageVolume> auxiliaryImageVolumes,
+      List<AuxiliaryImage> auxiliaryImages) {
+
+    Domain domainCR = CommonMiiTestUtils.createDomainResource(domainResourceName, domNamespace,
+        baseImageName, adminSecretName, repoSecretName,
+        encryptionSecretName, replicaCount, clusterName);
+
+    for (AuxiliaryImageVolume auxiliaryImageVolume : auxiliaryImageVolumes) {
+      domainCR.spec().addAuxiliaryImageVolumesItem(auxiliaryImageVolume);
+      domainCR.spec().configuration().model()
+          .withModelHome(auxiliaryImageVolume.getMountPath() + "/models")
+          .withWdtInstallHome(auxiliaryImageVolume.getMountPath() + "/weblogic-deploy");
+    }
+
+    for (AuxiliaryImage auxiliaryImage : auxiliaryImages) {
+      domainCR.spec().serverPod().addAuxiliaryImagesItem(auxiliaryImage);
+    }
+
+    return domainCR;
+  }
+
   /**
    * Create a domain object for a Kubernetes domain custom resource using the basic model-in-image
    * image.

integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java

@@ -28,6 +28,7 @@
 import io.kubernetes.client.custom.Quantity;
 import io.kubernetes.client.custom.V1Patch;
 import io.kubernetes.client.openapi.ApiException;
+import io.kubernetes.client.openapi.models.CoreV1Event;
 import io.kubernetes.client.openapi.models.NetworkingV1beta1HTTPIngressPath;
 import io.kubernetes.client.openapi.models.NetworkingV1beta1HTTPIngressRuleValue;
 import io.kubernetes.client.openapi.models.NetworkingV1beta1IngressBackend;
@@ -185,12 +186,14 @@
 import static oracle.weblogic.kubernetes.actions.TestActions.createService;
 import static oracle.weblogic.kubernetes.actions.TestActions.createServiceAccount;
 import static oracle.weblogic.kubernetes.actions.TestActions.defaultAppParams;
+import static oracle.weblogic.kubernetes.actions.TestActions.deleteDomainCustomResource;
 import static oracle.weblogic.kubernetes.actions.TestActions.dockerLogin;
 import static oracle.weblogic.kubernetes.actions.TestActions.dockerPush;
 import static oracle.weblogic.kubernetes.actions.TestActions.getJob;
 import static oracle.weblogic.kubernetes.actions.TestActions.getOperatorImageName;
 import static oracle.weblogic.kubernetes.actions.TestActions.getPersistentVolume;
 import static oracle.weblogic.kubernetes.actions.TestActions.getPersistentVolumeClaim;
+import static oracle.weblogic.kubernetes.actions.TestActions.getPod;
 import static oracle.weblogic.kubernetes.actions.TestActions.getPodCreationTimestamp;
 import static oracle.weblogic.kubernetes.actions.TestActions.getPodLog;
 import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort;
@@ -220,6 +223,7 @@
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.credentialsNotValid;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.credentialsValid;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.doesImageExist;
+import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainDoesNotExist;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainExists;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainStatusReasonMatches;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.isApacheReady;
@@ -247,6 +251,8 @@
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.serviceExists;
 import static oracle.weblogic.kubernetes.utils.ExecCommand.exec;
 import static oracle.weblogic.kubernetes.utils.FileUtils.checkDirectory;
+import static oracle.weblogic.kubernetes.utils.K8sEvents.checkDomainEvent;
+import static oracle.weblogic.kubernetes.utils.K8sEvents.getEvent;
 import static oracle.weblogic.kubernetes.utils.TestUtils.callWebAppAndCheckForServerNameInResponse;
 import static oracle.weblogic.kubernetes.utils.TestUtils.callWebAppAndWaitTillReady;
 import static oracle.weblogic.kubernetes.utils.TestUtils.getNextFreePort;
@@ -3866,6 +3872,27 @@ public static String getIntrospectJobName(String domainUid) {
     return domainUid + TestConstants.DEFAULT_INTROSPECTOR_JOB_NAME_SUFFIX;
   }
 
+  /**
+   * Get the introspector pod name.
+   * @param domainUid domain uid of the domain
+   * @param domainNamespace domain namespace in which introspector runs
+   * @return the introspector pod name
+   * @throws ApiException if Kubernetes API calls fail
+   */
+  public static String getIntrospectorPodName(String domainUid, String domainNamespace) throws ApiException {
+    checkPodExists(getIntrospectJobName(domainUid), domainUid, domainNamespace);
+
+    String labelSelector = String.format("weblogic.domainUID in (%s)", domainUid);
+
+    V1Pod introspectorPod = getPod(domainNamespace, labelSelector, getIntrospectJobName(domainUid));
+
+    if (introspectorPod != null && introspectorPod.getMetadata() != null) {
+      return introspectorPod.getMetadata().getName();
+    } else {
+      return "";
+    }
+  }
+
   /**
    * Set the inter-pod anti-affinity  for the domain custom resource
    * so that server instances spread over the available Nodes.
@@ -4352,4 +4379,116 @@ public static String getDockerExtraArgs() {
     }
     return extraArgs.toString();
   }
+
+  /**
+   * Wait until a given event is logged by the operator.
+   *
+   * @param opNamespace namespace in which the operator is running
+   * @param domainNamespace namespace in which the domain exists
+   * @param domainUid UID of the domain
+   * @param reason event to check for Created, Changed, deleted, processing etc
+   * @param type type of event, Normal of Warning
+   * @param timestamp the timestamp after which to see events
+   */
+  public static void checkEvent(
+      String opNamespace, String domainNamespace, String domainUid,
+      String reason, String type, OffsetDateTime timestamp) {
+    withStandardRetryPolicy
+        .conditionEvaluationListener(condition ->
+            getLogger().info("Waiting for domain event {0} to be logged in namespace {1} "
+                    + "(elapsed time {2}ms, remaining time {3}ms)",
+                reason,
+                domainNamespace,
+                condition.getElapsedTimeInMS(),
+                condition.getRemainingTimeInMS()))
+        .until(checkDomainEvent(opNamespace, domainNamespace, domainUid, reason, type, timestamp));
+  }
+
+  /**
+   * Delete a domain in the specified namespace.
+   * @param domainNS the namespace in which the domain exists
+   * @param domainUid domain uid
+   */
+  public static void deleteDomainResource(String domainNS, String domainUid) {
+    //clean up domain resources in namespace and set namespace to label , managed by operator
+    getLogger().info("deleting domain custom resource {0}", domainUid);
+    assertTrue(deleteDomainCustomResource(domainUid, domainNS));
+
+    // wait until domain was deleted
+    withStandardRetryPolicy
+        .conditionEvaluationListener(
+            condition -> getLogger().info("Waiting for domain {0} to be deleted in namespace {1} "
+                    + "(elapsed time {2}ms, remaining time {3}ms)",
+                domainUid,
+                domainNS,
+                condition.getElapsedTimeInMS(),
+                condition.getRemainingTimeInMS()))
+        .until(domainDoesNotExist(domainUid, DOMAIN_VERSION, domainNS));
+  }
+
+  private static Callable<Boolean> podLogContainsString(String namespace, String podName, String expectedString) {
+    return () -> {
+      String podLog;
+      try {
+        podLog = getPodLog(podName, namespace);
+        getLogger().info("pod log for pod {0} in namespace {1} : {2}", podName, namespace, podLog);
+      } catch (ApiException apiEx) {
+        getLogger().severe("got ApiException while getting pod log: ", apiEx);
+        return false;
+      }
+
+      return podLog.contains(expectedString);
+    };
+  }
+
+  /**
+   * Wait and check the pod log contains the expected string.
+   * @param namespace the namespace in which the pod exists
+   * @param podName the pod to get the log
+   * @param expectedString the expected string to check in the pod log
+   */
+  public static  void checkPodLogContainsString(String namespace, String podName, String expectedString) {
+
+    getLogger().info("Wait for string {0} existing in pod {1} in namespace {2}", expectedString, podName, namespace);
+    withStandardRetryPolicy
+        .conditionEvaluationListener(
+            condition -> getLogger().info("Waiting for string {0} existing in pod {1} in namespace {2} "
+                    + "(elapsed time {3}ms, remaining time {4}ms)",
+                expectedString,
+                podName,
+                namespace,
+                condition.getElapsedTimeInMS(),
+                condition.getRemainingTimeInMS()))
+        .until(assertDoesNotThrow(() -> podLogContainsString(namespace, podName, expectedString),
+            "podLogContainsString failed with IOException, ApiException or InterruptedException"));
+  }
+
+  /**
+   * Check the domain event contains the expected error msg.
+   *
+   * @param opNamespace namespace in which the operator is running
+   * @param domainNamespace namespace in which the domain exists
+   * @param domainUid UID of the domain
+   * @param reason event to check for Created, Changed, deleted, processing etc
+   * @param type type of event, Normal of Warning
+   * @param timestamp the timestamp after which to see events
+   * @param expectedMsg the expected message in the domain event message
+   */
+  public static void checkDomainEventContainsExpectedMsg(String opNamespace,
+                                                         String domainNamespace,
+                                                         String domainUid,
+                                                         String reason,
+                                                         String type,
+                                                         OffsetDateTime timestamp,
+                                                         String expectedMsg) {
+    checkEvent(opNamespace, domainNamespace, domainUid, reason, type, timestamp);
+    CoreV1Event event =
+        getEvent(opNamespace, domainNamespace, domainUid, reason, type, timestamp);
+    if (event != null && event.getMessage() != null) {
+      assertTrue(event.getMessage().contains(expectedMsg),
+          String.format("The event message does not contain the expected msg %s", expectedMsg));
+    } else {
+      fail("event is null or event message is null");
+    }
+  }
 }

integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/K8sEvents.java

@@ -159,7 +159,7 @@ public static boolean domainEventExists(
         }
       }
     } catch (ApiException ex) {
-      Logger.getLogger(ItKubernetesEvents.class.getName()).log(Level.SEVERE, null, ex);
+      logger.log(Level.SEVERE, null, ex);
     }
     return false;
   }

integration-tests/src/test/resources/auxiliaryimage/Dockerfile

@@ -17,7 +17,7 @@
 #
 
 FROM busybox
-ARG AUXILIARY_IMAGE_PATH=/common
+ARG AUXILIARY_IMAGE_PATH=/auxiliary
 ARG USER=oracle
 ARG USERID=1000
 ARG GROUP=root