Allow setting correct permissions for java certificates from quickstarters and/or agents

jenkins/agent-base/Dockerfile.centos7

@@ -11,14 +11,12 @@ ENV SONAR_SCANNER_VERSION=3.1.0.1141 \
     GIT_LFS_VERSION=2.6.1 \
     SKOPEO_VERSION=0.1.37-3 \
     OSTREE_VERSION=2018.5-1 \
-    JAVA_TOOL_OPTIONS="-XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Dsun.zip.disableMemoryMapping=true"
+    JAVA_TOOL_OPTIONS="-XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:-UseContainerSupport -Dsun.zip.disableMemoryMapping=true"
 
 ARG APP_DNS
 ARG SNYK_DISTRIBUTION_URL
 ARG AQUASEC_SCANNERCLI_URL
 
-ENV JAVA_HOME=/usr/lib/jvm/jre
-
 RUN rm -fv /etc/yum.repos.d/CentOS-Media.repo /etc/yum.repos.d/origin-local-release.repo \
     && yum -y install openssl \
     && yum -y install java-11-openjdk-devel \
@@ -38,9 +36,9 @@ RUN chmod +x /usr/local/bin/use-j*.sh && \
     use-j11.sh && \
     echo "--- ENDS JDK 11 TESTS ---"
 
-
 COPY ./import_certs.sh /usr/local/bin/import_certs.sh
-RUN import_certs.sh
+COPY ./fix_java_certs_permissions.sh /usr/local/bin/fix_java_certs_permissions.sh
+RUN import_certs.sh && fix_java_certs_permissions.sh
 
 # Install Sonar Scanner.
 RUN cd /tmp \

jenkins/agent-base/Dockerfile.ubi8

@@ -9,7 +9,7 @@ ENV SONAR_SCANNER_VERSION=3.1.0.1141 \
     HELM_PLUGIN_DIFF_VERSION=3.3.2 \
     HELM_PLUGIN_SECRETS_VERSION=3.3.5 \
     GIT_LFS_VERSION=2.6.1 \
-    JAVA_TOOL_OPTIONS="-XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Dsun.zip.disableMemoryMapping=true"
+    JAVA_TOOL_OPTIONS="-XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:-UseContainerSupport -Dsun.zip.disableMemoryMapping=true"
 
 ARG APP_DNS
 ARG SNYK_DISTRIBUTION_URL
@@ -39,7 +39,8 @@ RUN chmod +x /usr/local/bin/use-j*.sh && \
     echo "--- ENDS JDK 11 TESTS ---"
 
 COPY ./import_certs.sh /usr/local/bin/import_certs.sh
-RUN import_certs.sh
+COPY ./fix_java_certs_permissions.sh /usr/local/bin/fix_java_certs_permissions.sh
+RUN import_certs.sh && fix_java_certs_permissions.sh
 
 # Install Sonar Scanner.
 RUN cd /tmp \

jenkins/agent-base/fix_java_certs_permissions.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -eu
+
+# Initialize JAVA_HOME if not set.
+JAVA_HOME=${JAVA_HOME:-""}
+
+if [ -f /etc/profile.d/set-default-java.sh ]; then
+    source /etc/profile.d/set-default-java.sh
+else
+    echo "WARNING: Not setting default java version."
+fi
+
+echo "Trying to setup correct permissions for cacerts folder... "
+if [ ! -z "${JAVA_HOME}" ] && [ "" != "${JAVA_HOME}" ]; then
+    chown -c 1001:0 $JAVA_HOME/lib/security/cacerts
+    chmod -c g+w $JAVA_HOME/lib/security/cacerts
+else
+    echo "WARNING: Cannot apply permissions 'chmod g+w' to JAVA_HOME/lib/security/cacerts "
+    echo "WARNING: JAVA_HOME=${JAVA_HOME}"
+fi

jenkins/agent-base/import_certs.sh

@@ -32,12 +32,3 @@ if [[ ! -z ${APP_DNS:=""} ]]; then
 else
     echo 'No certificates to import'
 fi
-
-echo "Trying to setup correct permissions for cacerts folder... "
-if [ ! -z "${JAVA_HOME}" ] && [ "" != "${JAVA_HOME}" ]; then
-    chown -c 1001:0 $JAVA_HOME/lib/security/cacerts
-    chmod -c g+w $JAVA_HOME/lib/security/cacerts
-else
-    echo "WARNING: Cannot apply permissions 'chmod g+w' to JAVA_HOME/lib/security/cacerts "
-    echo "WARNING: JAVA_HOME=${JAVA_HOME}"
-fi

jenkins/master/Dockerfile.centos7

@@ -25,6 +25,7 @@ RUN /usr/local/bin/install-plugins.sh /opt/openshift/configuration/plugins.txt \
     && mv /usr/libexec/s2i/run /usr/libexec/s2i/openshift-run
 COPY configuration/ /opt/openshift/configuration/
 COPY ods-run.sh /usr/libexec/s2i/run
+COPY logging.properties /var/lib/jenkins/
 
 RUN chown :0 /etc/pki/java/cacerts && chmod ugo+w /etc/pki/java/cacerts
 

jenkins/master/Dockerfile.rhel7

@@ -25,6 +25,7 @@ RUN /usr/local/bin/install-plugins.sh /opt/openshift/configuration/plugins.txt \
     && mv /usr/libexec/s2i/run /usr/libexec/s2i/openshift-run
 COPY configuration/ /opt/openshift/configuration/
 COPY ods-run.sh /usr/libexec/s2i/run
+COPY logging.properties /var/lib/jenkins/
 
 RUN chown :0 /etc/pki/java/cacerts && chmod ugo+w /etc/pki/java/cacerts
 

jenkins/master/Dockerfile.ubi8

@@ -25,6 +25,7 @@ RUN /usr/local/bin/install-plugins.sh /opt/openshift/configuration/plugins.txt \
     && mv /usr/libexec/s2i/run /usr/libexec/s2i/openshift-run
 COPY configuration/ /opt/openshift/configuration/
 COPY ods-run.sh /usr/libexec/s2i/run
+COPY logging.properties /var/lib/jenkins/
 
 RUN chown :0 /etc/pki/java/cacerts && chmod ugo+w /etc/pki/java/cacerts
 

jenkins/master/logging.properties

@@ -0,0 +1,11 @@
+handlers=java.util.logging.ConsoleHandler,java.util.logging.FileHandler
+
+java.util.logging.FileHandler.level=INFO
+java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter
+java.util.logging.FileHandler.pattern=/var/log/jenkins/jenkins-master.log
+java.util.logging.FileHandler.append=true
+java.util.logging.FileHandler.limit=10000000
+java.util.logging.FileHandler.count=5
+
+java.util.logging.ConsoleHandler.level=INFO
+java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter

jenkins/ocp-config/build/bc.yml

@@ -72,7 +72,7 @@ objects:
     labels:
       app: jenkins
   spec:
-    failedBuildsHistoryLimit: 5
+    failedBuildsHistoryLimit: 20
     nodeSelector: null
     output:
       to:
@@ -117,15 +117,15 @@ objects:
         from:
           kind: DockerImage
           name: ${JENKINS_MASTER_BASE_FROM_IMAGE}
-    successfulBuildsHistoryLimit: 5
+    successfulBuildsHistoryLimit: 20
 - kind: BuildConfig
   apiVersion: v1
   metadata:
     name: jenkins-agent-base
     labels:
       app: jenkins
   spec:
-    failedBuildsHistoryLimit: 5
+    failedBuildsHistoryLimit: 20
     nodeSelector: null
     output:
       to:
@@ -164,7 +164,7 @@ objects:
           name: ${JENKINS_AGENT_BASE_FROM_IMAGE}
         dockerfilePath: ${JENKINS_AGENT_DOCKERFILE_PATH}
       type: Docker
-    successfulBuildsHistoryLimit: 5
+    successfulBuildsHistoryLimit: 20
 - apiVersion: v1
   kind: BuildConfig
   metadata:

jenkins/ocp-config/deploy/jenkins-master.yml

@@ -17,7 +17,7 @@ parameters:
 - name: JENKINS_ENABLE_OAUTH
   value: "true"
 - name: JENKINS_MEMORY_LIMIT
-  value: 6Gi
+  value: 7Gi
 - name: JENKINS_MEMORY_REQUEST
   value: 4Gi
 - name: JENKINS_CPU_LIMIT
@@ -27,7 +27,7 @@ parameters:
 - name: JENKINS_VOLUME_CAPACITY
   value: 5Gi
 - name: JENKINS_JAVA_GC_OPTS
-  value: "-server -XX:NativeMemoryTracking=summary -XX:MaxRAMPercentage=90 -XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication -XX:MaxMetaspaceSize=1g -XX:MetaspaceSize=256M -XX:SoftRefLRUPolicyMSPerMB=1 -XX:+UnlockExperimentalVMOptions -XX:+UnlockDiagnosticVMOptions -XX:G1SummarizeRSetStatsPeriod=1"
+  value: "-server -XX:NativeMemoryTracking=summary -XX:-UseContainerSupport -XX:MaxRAMPercentage=90 -XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication -XX:MaxMetaspaceSize=1g -XX:MetaspaceSize=256M -XX:SoftRefLRUPolicyMSPerMB=1 -XX:+UnlockExperimentalVMOptions -XX:+UnlockDiagnosticVMOptions -XX:G1SummarizeRSetStatsPeriod=1 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/lib/jenkins"
 - name: JENKINS_JAVA_MAX_HEAP_PARAM
   value: "-Xms1024m -Xmx4g"
 - name: JENKINS_CONTAINER_HEAP_PERCENT

ods-devenv/scripts/deploy.sh

@@ -1574,10 +1574,13 @@ function wait_until_http_svc_is_up() {
     local SVC_HTTP_URL="${2}"
     local CURL_SVC_OUTPUT_FILE="/tmp/result-curl-svc-${SVC_NAME}-output"
     local CURL_SVC_HEADERS_FILE="/tmp/result-curl-svc-${SVC_NAME}-headers"
+    local CURL_LOGS_CHECK_SVC_FILE="/tmp/result-curl-svc-${SVC_NAME}-curlresult"
     local retryMax=${3:-20}
 
-    wait_until_http_svc_is_up_advanced "$SVC_NAME" "$SVC_HTTP_URL" "$CURL_SVC_OUTPUT_FILE" "$CURL_SVC_HEADERS_FILE" $retryMax
-    if [ 0 -ne $? ]; then
+    local RETURN_VALUE=0
+    wait_until_http_svc_is_up_advanced "$SVC_NAME" "$SVC_HTTP_URL" "$CURL_SVC_OUTPUT_FILE" "$CURL_SVC_HEADERS_FILE" \
+        "${CURL_LOGS_CHECK_SVC_FILE}" $retryMax || RETURN_VALUE=1
+    if [ 0 -ne ${RETURN_VALUE} ]; then
         echo "[STATUS CHECK] ERROR: Service is down and we cannot live without it: ${SVC_NAME}"
         return 1
     fi
@@ -1589,7 +1592,8 @@ function wait_until_http_svc_is_up_advanced() {
     local SVC_HTTP_URL="${2}"
     local CURL_SVC_OUTPUT_FILE="${3}"
     local CURL_SVC_HEADERS_FILE="${4}"
-    local retryMaxIn=${5:-20}
+    local CURL_LOGS_CHECK_SVC_FILE="${5}"
+    local retryMaxIn=${6:-20}
     local retryMax=$((retryMaxIn))
 
     echo " "
@@ -1620,14 +1624,18 @@ function wait_until_http_svc_is_up_advanced() {
         fi
 
         # Remove files from previous execution.
-        rm -fv ${CURL_SVC_OUTPUT_FILE} ${CURL_SVC_HEADERS_FILE} || true
+        rm -fv ${CURL_SVC_OUTPUT_FILE} ${CURL_SVC_HEADERS_FILE} ${CURL_LOGS_CHECK_SVC_FILE} || true
         if [ -f ${CURL_SVC_OUTPUT_FILE} ] || [ -f ${CURL_SVC_HEADERS_FILE} ]; then
             echo "[STATUS CHECK] WARNING: Could NOT remove files ${CURL_SVC_OUTPUT_FILE} ${CURL_SVC_HEADERS_FILE} "
         fi
 
-        if ! curl --insecure -sSL --retry-delay 2 --retry-max-time 20 --retry 10 --dump-header ${CURL_SVC_HEADERS_FILE} ${SVC_HTTP_URL} -o ${CURL_SVC_OUTPUT_FILE} ; then
+        local CURL_RETURN_VAL=0
+        curl --insecure -sSL --retry-delay 2 --retry-max-time 20 --retry 10 --dump-header ${CURL_SVC_HEADERS_FILE} \
+                -o ${CURL_SVC_OUTPUT_FILE} ${SVC_HTTP_URL} 2>&1 > ${CURL_LOGS_CHECK_SVC_FILE} || CURL_RETURN_VAL=1
+        if [ 0 -ne ${CURL_RETURN_VAL} ]; then
             echo "Curl replied != 0 for query to ${SVC_HTTP_URL} "
             echo "Checking if it was caused by a redirect... "
+            grep -i 'HTTP' ${CURL_LOGS_CHECK_SVC_FILE} || true
         fi
 
         if ! grep -q '^\s*HTTP/[0-9\.]*\s*200[\s]*' ${CURL_SVC_HEADERS_FILE} ; then
@@ -2703,9 +2711,12 @@ function check_pods_and_restart_if_necessary() {
     local retryMaxIn=${1:-5}
     local retryMaxHttpIn=${2:-10}
 
-    check_pod_and_restart_if_necessary 'sonarqube' 'ods/sonarqube' 'https://sonarqube-ods.ocp.odsbox.lan/' ${retryMaxIn} ${retryMaxHttpIn}
-    check_pod_and_restart_if_necessary 'prov-app' 'ods/ods-provisioning-app' 'https://prov-app-ods.ocp.odsbox.lan/' ${retryMaxIn} ${retryMaxHttpIn}
-    check_pod_and_restart_if_necessary 'nexus' 'ods/nexus' 'https://nexus-ods.ocp.odsbox.lan/' ${retryMaxIn} ${retryMaxHttpIn}
+    check_pod_and_restart_if_necessary 'sonarqube' 'ods/sonarqube' 'https://sonarqube-ods.ocp.odsbox.lan/' \
+        ${retryMaxIn} ${retryMaxHttpIn} || restart_ods
+    check_pod_and_restart_if_necessary 'prov-app' 'ods/ods-provisioning-app' 'https://prov-app-ods.ocp.odsbox.lan/' \
+        ${retryMaxIn} ${retryMaxHttpIn} || restart_ods
+    check_pod_and_restart_if_necessary 'nexus' 'ods/nexus' 'https://nexus-ods.ocp.odsbox.lan/' \
+        ${retryMaxIn} ${retryMaxHttpIn} || restart_ods
     # https://jenkins-ods.ocp.odsbox.lan
 }
 
@@ -2716,6 +2727,7 @@ function check_pod_and_restart_if_necessary() {
     local SVC_HTTP_URL="${3}"
     local CURL_SVC_OUTPUT_FILE="/tmp/result-curl-svc-${SVC_NAME}-output"
     local CURL_SVC_HEADERS_FILE="/tmp/result-curl-svc-${SVC_NAME}-headers"
+    local CURL_LOGS_CHECK_SVC_FILE="/tmp/result-curl-svc-${SVC_NAME}-curlresult"
     local retryMaxIn=${4:-5}
     local retryMax=$((retryMaxIn))
     local retryMaxHttpIn=${5:-10}
@@ -2735,7 +2747,8 @@ function check_pod_and_restart_if_necessary() {
         fi
 
 	    retVal=0
-        wait_until_http_svc_is_up_advanced "$SVC_NAME" "$SVC_HTTP_URL" "$CURL_SVC_OUTPUT_FILE" "$CURL_SVC_HEADERS_FILE" ${retryMaxHttpIn} || retVal=1
+        wait_until_http_svc_is_up_advanced "$SVC_NAME" "$SVC_HTTP_URL" "$CURL_SVC_OUTPUT_FILE" "$CURL_SVC_HEADERS_FILE" \
+            ${CURL_LOGS_CHECK_SVC_FILE} ${retryMaxHttpIn} || retVal=1
 
         if [ 0 -ne ${retVal} ]; then
             echo "[STATUS CHECK] WARNING: Stopping pod so it restarts automatically. Service: ${SVC_NAME} "
@@ -2749,7 +2762,10 @@ function check_pod_and_restart_if_necessary() {
 
             if [ "false" == "$docker_process_killed" ]; then
                 echo "No docker process found for pod ${SVC_NAME} with ID ${SVC_POD_ID} "
+                echo "Current docker pods: "
+                docker ps -a | grep -v 'Exited .* ago' || true
                 echo " "
+                return 1
             fi
         fi
 

tests/scripts/free-unused-resources.sh

@@ -2,6 +2,8 @@
 
 echo " "
 
+ME=$(basename $0)
+
 function clean_containers {
 	echo "Removing docker containers no more used... "
 	if docker ps -a | grep -q 'Exited .* ago' ; then
@@ -27,6 +29,7 @@ function clean_tests {
 }
 
 function clean_odsverify {
+	echo "Cleaning projects ODS__VERIFY... "
 	if [ "true" == "$CLEAN_ODS_VERIFY" ]; then
 		echo "Removing ODS VERIFY projects..."
 		oc projects | grep '^\s*odsverify.*' | while read -r line; do
@@ -41,14 +44,14 @@ function clean_odsverify {
 }
 
 function clean_images {
+    echo "Cleaning OC images"
     echo "oc adm prune images --keep-tag-revisions=1 --keep-younger-than=30m --confirm"
 	oc adm prune images --keep-tag-revisions=1 --keep-younger-than=30m --confirm || true
 }
 
 function usage {
-	ME=$(basename $0)
 	echo " "
-	echo "usage: ${ME} [--odsVerify] [--omitTestsProject tes22]"
+	echo "usage: ${ME} [--odsVerify] [--omitTests] [--omitTestsProject tes22]"
 	echo " "
 }
 
@@ -59,6 +62,7 @@ function echo_error() {
 
 OMIT_TESTS_PROJECT=none
 CLEAN_ODS_VERIFY="false"
+CLEAN_TESTS="false"
 
 while [[ "$#" -gt 0 ]]; do
     case $1 in
@@ -71,11 +75,19 @@ while [[ "$#" -gt 0 ]]; do
 
     --omitTestsProject) OMIT_TESTS_PROJECT="$2"; echo "Tests to omit: $OMIT_TESTS_PROJECT"; shift;;
 
+    --cleanTests) CLEAN_TESTS="true";;
+
   *) echo_error "Unknown parameter passed: $1"; exit 1;;
 esac; shift; done
 
 clean_containers
-clean_tests
+if [ "true" == "${CLEAN_TESTS}" ]; then
+	clean_tests
+else
+	echo " "
+	echo "${ME}: INFO: Not cleaning tests"
+	echo " "
+fi
 clean_odsverify
 clean_images