Show Jenkins master/agent errors in the boxes, when they do not work as expected.

CHANGELOG.md

@@ -30,6 +30,7 @@
 - Fix CI/CD problems in Jenkins pipelines ([#1177](https://github.com/opendevstack/ods-core/pull/1177))
 - Fixes Python agent does not seems to have java in the path ([#685](https://github.com/opendevstack/ods-quickstarters/issues/685)) 
 - Removes existing differences between jenkins agent base image in Centos 7 and UBI 8 ([#1181](https://github.com/opendevstack/ods-core/pull/1181))
+- Upgrade to Java 11 Jenkins master and agents and increase logging to know why they sometimes die ([#1185](https://github.com/opendevstack/ods-core/pull/1185))
 
 ### Added
 

jenkins/agent-base/Dockerfile.centos7

@@ -17,11 +17,10 @@ ARG APP_DNS
 ARG SNYK_DISTRIBUTION_URL
 ARG AQUASEC_SCANNERCLI_URL
 
+COPY ensure_java_jre_is_adequate.sh /usr/local/bin/
 RUN rm -fv /etc/yum.repos.d/CentOS-Media.repo /etc/yum.repos.d/origin-local-release.repo \
+    && ensure_java_jre_is_adequate.sh \
     && yum -y install openssl \
-    && yum -y install java-11-openjdk-devel \
-    && yum list installed | grep java \
-    && yum -y remove java-1.8* \
     && yum -y update \
     && yum clean all \
     && rm -rf /var/cache/yum/*
@@ -115,14 +114,9 @@ RUN . /tmp/set_java_proxy.sh && echo $JAVA_OPTS
 # It is setting the variable JAVA_TOOL_OPTIONS while it should not.
 # Besides, we need to know if this variable has not been set.
 # It is a problem very difficult to detect...
+COPY fix_openshift_run_jnlp_client.sh /usr/local/bin/fix_openshift_run_jnlp_client.sh
 RUN mv /usr/local/bin/run-jnlp-client /usr/local/bin/openshift-run-jnlp-client \
-    && sed -i 's|\#\!/bin/bash|\#\!/bin/bash -x|g' /usr/local/bin/openshift-run-jnlp-client \
-    && sed -i "s|^\s*JAVA_TOOL_OPTIONS\s*=.*|    echo 'WARNING: JAVA_TOOL_OPTIONS env variable is UNSET.'|g" \
-        /usr/local/bin/openshift-run-jnlp-client \
-    && sed -i "s|^\s*export\s*JAVA_TOOL_OPTIONS.*|    echo 'WARNING: JAVA_TOOL_OPTIONS env variable is UNSET.'|g" \
-        /usr/local/bin/openshift-run-jnlp-client \
-    && sed -i 's|^\(\s*\)JAVA_GC_OPTS\s*=.*|\1JAVA_GC_OPTS=|g' /usr/local/bin/openshift-run-jnlp-client \
-    && grep -B 3 -A 3 -i '\(bash\|JAVA_TOOL_OPTIONS\|JAVA_GC_OPTS\)' /usr/local/bin/openshift-run-jnlp-client
+    && fix_openshift_run_jnlp_client.sh /usr/local/bin/openshift-run-jnlp-client
 
 COPY ods-run-jnlp-client.sh /usr/local/bin/run-jnlp-client
 

jenkins/agent-base/Dockerfile.ubi8

@@ -18,7 +18,9 @@ ARG AQUASEC_SCANNERCLI_URL
 # Add UBI repositories.
 COPY yum.repos.d/ubi.repo /etc/yum.repos.d/ubi.repo
 
+COPY ensure_java_jre_is_adequate.sh /usr/local/bin/
 RUN cd /etc/yum.repos.d && rm -f localdev-* ci-rpm-mirrors.repo \
+    && ensure_java_jre_is_adequate.sh \
     && yum -y install make glibc-langpack-en openssl \
     && yum -y update \
     && yum clean all \
@@ -115,14 +117,9 @@ COPY set_java_proxy.sh /tmp/set_java_proxy.sh
 RUN . /tmp/set_java_proxy.sh && echo $JAVA_OPTS
 
 # Customize entrypoint.
+COPY fix_openshift_run_jnlp_client.sh /usr/local/bin/fix_openshift_run_jnlp_client.sh
 RUN mv /usr/local/bin/run-jnlp-client /usr/local/bin/openshift-run-jnlp-client \
-    && sed -i 's|\#\!/bin/bash|\#\!/bin/bash -x|g' /usr/local/bin/openshift-run-jnlp-client \
-    && sed -i "s|^\s*JAVA_TOOL_OPTIONS\s*=.*|    echo 'WARNING: JAVA_TOOL_OPTIONS env variable is UNSET.'|g" \
-        /usr/local/bin/openshift-run-jnlp-client \
-    && sed -i "s|^\s*export\s*JAVA_TOOL_OPTIONS.*|    echo 'WARNING: JAVA_TOOL_OPTIONS env variable is UNSET.'|g" \
-        /usr/local/bin/openshift-run-jnlp-client \
-    && sed -i 's|^\(\s*\)JAVA_GC_OPTS\s*=.*|\1JAVA_GC_OPTS=|g' /usr/local/bin/openshift-run-jnlp-client \
-    && grep -B 3 -A 3 -i '\(bash\|JAVA_TOOL_OPTIONS\|JAVA_GC_OPTS\)' /usr/local/bin/openshift-run-jnlp-client
+    && fix_openshift_run_jnlp_client.sh /usr/local/bin/openshift-run-jnlp-client
 
 COPY ods-run-jnlp-client.sh /usr/local/bin/run-jnlp-client
 

jenkins/agent-base/ensure_java_jre_is_adequate.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+set -eu -o pipefail
+
+ME="$(basename $0)"
+JAVA_INSTALLED_PKGS_LOGS="/tmp/java_installed_pkgs.log"
+JAVA_11_INSTALLED_PKGS_LOGS="/tmp/java_11_installed_pkgs.log"
+rm -fv ${JAVA_INSTALLED_PKGS_LOGS} ${JAVA_11_INSTALLED_PKGS_LOGS}
+
+NEEDS_DEVEL=${1-""}
+PKG_NAME_TAIL="headless"
+if [ ! -z "${NEEDS_DEVEL}" ] && [ "" != "${NEEDS_DEVEL}" ]; then
+    NEEDS_DEVEL="true"
+    PKG_NAME_TAIL="devel"
+else
+    NEEDS_DEVEL="false"
+    PKG_NAME_TAIL="headless"
+fi
+
+echo "${ME}: Needs development packages? ${NEEDS_DEVEL}"
+echo " "
+echo "${ME}: Listing versions of java installed: "
+yum list installed | grep -i "\(java\|jre\)" | tee -a ${JAVA_INSTALLED_PKGS_LOGS}
+touch ${JAVA_11_INSTALLED_PKGS_LOGS}
+grep -i "java-11" ${JAVA_INSTALLED_PKGS_LOGS} > ${JAVA_11_INSTALLED_PKGS_LOGS} || echo "No java 11 packages found."
+
+NEEDS_INSTALLATION="true"
+if [ -f ${JAVA_11_INSTALLED_PKGS_LOGS} ]; then
+    if grep -qi "${PKG_NAME_TAIL}" ${JAVA_11_INSTALLED_PKGS_LOGS} ; then
+        NEEDS_INSTALLATION="false"
+    fi
+fi
+
+# We need devel package in masters to have jar binary.
+if [ "true" == "${NEEDS_INSTALLATION}" ]; then
+    echo "${ME}:Java-11 is *not* installed. Installing..."
+    if [ "true" == "${NEEDS_DEVEL}" ]; then
+        yum -y install java-11-openjdk-devel
+    else
+        yum -y install java-11-openjdk-headless
+    fi
+else
+    echo "${ME}: Java-11 is already installed."
+fi
+
+if grep -qi "java-1.8" ${JAVA_INSTALLED_PKGS_LOGS} ; then
+    echo "${ME}: Java-8 is installed. Removing..."
+    yum -y remove java-1.8*
+else
+    echo "${ME}: Java-8 is not installed. Correct."
+fi
+
+rm -fv ${JAVA_INSTALLED_PKGS_LOGS} ${JAVA_11_INSTALLED_PKGS_LOGS}
+
+echo " "
+echo "${ME}: Checking java tool versions: "
+if [ "true" == "${NEEDS_DEVEL}" ]; then
+    jar --version
+fi
+
+NO_JAVA_LINK="false"
+java -version || NO_JAVA_LINK="true"
+if [ "true" == "${NO_JAVA_LINK}" ]; then
+    JAVA_HOME_FOLDER=$(ls -lah /usr/lib/jvm | grep "java-11-openjdk-11.*\.x86_64" | awk '{print $NF}' | head -1)
+    JAVA_HOME="/usr/lib/jvm/${JAVA_HOME_FOLDER}"
+    alternatives --set java ${JAVA_HOME}/bin/java
+fi
+java -version

jenkins/agent-base/fix_openshift_run_jnlp_client.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -eu -o pipefail
+
+FILEPATH=${1-"/usr/local/bin/openshift-run-jnlp-client"}
+
+if [ ! -f ${FILEPATH} ]; then
+    echo " "
+    echo "ERROR: File does not exist: ${FILEPATH}"
+    echo " "
+    exit 1
+fi
+
+sed -i 's|\#\!/bin/bash|\#\!/bin/bash -x|g' ${FILEPATH}
+sed -i "s|^\s*JAVA_TOOL_OPTIONS\s*=.*|    echo 'WARNING: JAVA_TOOL_OPTIONS env variable is UNSET.'|g" \
+    ${FILEPATH}
+sed -i "s|^\s*export\s*JAVA_TOOL_OPTIONS.*|    echo 'WARNING: JAVA_TOOL_OPTIONS env variable is UNSET.'|g" \
+    ${FILEPATH}
+sed -i 's|^\(\s*\)JAVA_GC_OPTS\s*=.*|\1JAVA_GC_OPTS=|g' ${FILEPATH}
+sed -i 's|curl\s*-sS\s*|curl -sSLv |g' ${FILEPATH}
+
+grep -B 3 -A 3 -i '\(bash\|JAVA_TOOL_OPTIONS\|JAVA_GC_OPTS\|curl\)' ${FILEPATH}
+
+

jenkins/master/Dockerfile.centos7

@@ -14,7 +14,20 @@ ENV TAILOR_VERSION=1.3.4
 USER root
 
 COPY ./import_certs.sh /usr/local/bin/import_certs.sh
-RUN import_certs.sh
+COPY ./ensure_java_jre_is_adequate.sh /usr/local/bin/ensure_java_jre_is_adequate.sh
+COPY ./fix_openshift_scripts.sh /usr/local/bin/fix_openshift_scripts.sh
+COPY ./clean_yum_cache.sh /usr/local/bin/clean_yum_cache.sh
+RUN import_certs.sh \
+    && rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key \
+    && ensure_java_jre_is_adequate.sh master \
+    && fix_openshift_scripts.sh \
+    && yum -y update \
+    && clean_yum_cache.sh
+
+
+# To debug. Warning: too much logs if we enable it.
+#RUN sed -i 's|\#\!\s*/bin/bash.*|\#\!/bin/bash -xeu|g' /usr/local/bin/install-plugins.sh \
+#    && head -n 10 /usr/local/bin/install-plugins.sh
 
 # Copy configuration and plugins.
 COPY plugins.centos7.txt /opt/openshift/configuration/plugins.txt
@@ -31,7 +44,7 @@ RUN chown :0 /etc/pki/java/cacerts && chmod ugo+w /etc/pki/java/cacerts
 
 # Install Tailor.
 RUN cd /tmp \
-	&& curl -LOv https://github.com/opendevstack/tailor/releases/download/v${TAILOR_VERSION}/tailor-linux-amd64 \
+	&& curl -sSLO https://github.com/opendevstack/tailor/releases/download/v${TAILOR_VERSION}/tailor-linux-amd64 \
 	&& mv tailor-linux-amd64 /usr/local/bin/tailor \
 	&& chmod a+x /usr/local/bin/tailor
 

jenkins/master/Dockerfile.rhel7

@@ -14,7 +14,14 @@ ENV TAILOR_VERSION=1.3.4
 USER root
 
 COPY ./import_certs.sh /usr/local/bin/import_certs.sh
-RUN import_certs.sh
+COPY ./ensure_java_jre_is_adequate.sh /usr/local/bin/ensure_java_jre_is_adequate.sh
+COPY ./fix_openshift_scripts.sh /usr/local/bin/fix_openshift_scripts.sh
+COPY ./clean_yum_cache.sh /usr/local/bin/clean_yum_cache.sh
+RUN import_certs.sh \
+    && rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key \
+    && ensure_java_jre_is_adequate.sh master \
+    && fix_openshift_scripts.sh \
+    && clean_yum_cache.sh
 
 # Copy configuration and plugins.
 COPY plugins.rhel7.txt /opt/openshift/configuration/plugins.txt

jenkins/master/Dockerfile.ubi8

@@ -14,7 +14,17 @@ ENV TAILOR_VERSION=1.3.4
 USER root
 
 COPY ./import_certs.sh /usr/local/bin/import_certs.sh
-RUN import_certs.sh
+COPY ./ensure_java_jre_is_adequate.sh /usr/local/bin/ensure_java_jre_is_adequate.sh
+COPY ./fix_openshift_scripts.sh /usr/local/bin/fix_openshift_scripts.sh
+COPY ./clean_yum_cache.sh /usr/local/bin/clean_yum_cache.sh
+RUN import_certs.sh \
+    && rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key \
+    && sed -i 's|enabled\s*=\s*1|enabled=0|g' /etc/yum.repos.d/ci-rpm-mirrors.repo \
+    && sed -i 's|enabled\s*=\s*1|enabled=0|g' /etc/yum.repos.d/localdev-* \
+    && sed -i 's|enabled\s*=\s*1|enabled=0|g' /etc/yum.repos.d/epel.repo \
+    && ensure_java_jre_is_adequate.sh master \
+    && fix_openshift_scripts.sh \
+    && clean_yum_cache.sh
 
 # Copy configuration and plugins.
 COPY plugins.ubi8.txt /opt/openshift/configuration/plugins.txt

jenkins/master/clean_yum_cache.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -eu -o pipefail
+
+ME="$(basename $0)"
+
+yum clean all || true
+rm -rf /var/cache/yum/* || true

jenkins/master/ensure_java_jre_is_adequate.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+set -eu -o pipefail
+
+ME="$(basename $0)"
+JAVA_INSTALLED_PKGS_LOGS="/tmp/java_installed_pkgs.log"
+JAVA_11_INSTALLED_PKGS_LOGS="/tmp/java_11_installed_pkgs.log"
+rm -fv ${JAVA_INSTALLED_PKGS_LOGS} ${JAVA_11_INSTALLED_PKGS_LOGS}
+
+NEEDS_DEVEL=${1-""}
+PKG_NAME_TAIL="headless"
+if [ ! -z "${NEEDS_DEVEL}" ] && [ "" != "${NEEDS_DEVEL}" ]; then
+    NEEDS_DEVEL="true"
+    PKG_NAME_TAIL="devel"
+else
+    NEEDS_DEVEL="false"
+    PKG_NAME_TAIL="headless"
+fi
+
+echo "${ME}: Needs development packages? ${NEEDS_DEVEL}"
+echo " "
+echo "${ME}: Listing versions of java installed: "
+yum list installed | grep -i "\(java\|jre\)" | tee -a ${JAVA_INSTALLED_PKGS_LOGS}
+touch ${JAVA_11_INSTALLED_PKGS_LOGS}
+grep -i "java-11" ${JAVA_INSTALLED_PKGS_LOGS} > ${JAVA_11_INSTALLED_PKGS_LOGS} || echo "No java 11 packages found."
+
+NEEDS_INSTALLATION="true"
+if [ -f ${JAVA_11_INSTALLED_PKGS_LOGS} ]; then
+    if grep -qi "${PKG_NAME_TAIL}" ${JAVA_11_INSTALLED_PKGS_LOGS} ; then
+        NEEDS_INSTALLATION="false"
+    fi
+fi
+
+# We need devel package in masters to have jar binary.
+if [ "true" == "${NEEDS_INSTALLATION}" ]; then
+    echo "${ME}:Java-11 is *not* installed. Installing..."
+    if [ "true" == "${NEEDS_DEVEL}" ]; then
+        yum -y install java-11-openjdk-devel
+    else
+        yum -y install java-11-openjdk-headless
+    fi
+else
+    echo "${ME}: Java-11 is already installed."
+fi
+
+if grep -qi "java-1.8" ${JAVA_INSTALLED_PKGS_LOGS} ; then
+    echo "${ME}: Java-8 is installed. Removing..."
+    yum -y remove java-1.8*
+else
+    echo "${ME}: Java-8 is not installed. Correct."
+fi
+
+rm -fv ${JAVA_INSTALLED_PKGS_LOGS} ${JAVA_11_INSTALLED_PKGS_LOGS}
+
+echo " "
+echo "${ME}: Checking java tool versions: "
+if [ "true" == "${NEEDS_DEVEL}" ]; then
+    jar --version
+fi
+
+NO_JAVA_LINK="false"
+java -version || NO_JAVA_LINK="true"
+if [ "true" == "${NO_JAVA_LINK}" ]; then
+    JAVA_HOME_FOLDER=$(ls -lah /usr/lib/jvm | grep "java-11-openjdk-11.*\.x86_64" | awk '{print $NF}' | head -1)
+    JAVA_HOME="/usr/lib/jvm/${JAVA_HOME_FOLDER}"
+    alternatives --set java ${JAVA_HOME}/bin/java
+fi
+java -version

jenkins/master/fix_openshift_scripts.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -eu -o pipefail
+
+ME="$(basename $0)"
+echo "${ME}: INFO: Fixing openshift scripts..."
+
+FILE_TO_MODIFY=${1-"/usr/libexec/s2i/run"}
+
+if [ -f ${FILE_TO_MODIFY} ]; then
+    sed -i 's|\#\!/bin/bash|\#\!/bin/bash -x|g' ${FILE_TO_MODIFY}
+    sed -i "s|^\s*JAVA_TOOL_OPTIONS\s*=.*|    echo 'WARNING: JAVA_TOOL_OPTIONS env variable is UNSET.'|g" \
+            ${FILE_TO_MODIFY}
+    sed -i "s|^\s*export\s*JAVA_TOOL_OPTIONS.*|    echo 'WARNING: JAVA_TOOL_OPTIONS env variable is UNSET.'|g" \
+            ${FILE_TO_MODIFY}
+    sed -i 's|^\(\s*\)JAVA_GC_OPTS\s*=.*|\1JAVA_GC_OPTS=|g' ${FILE_TO_MODIFY}
+    grep -B 3 -A 3 -i '\(bash\|JAVA_TOOL_OPTIONS\|JAVA_GC_OPTS\)' ${FILE_TO_MODIFY}
+else
+    echo " "
+    echo "${ME}: WARNING: Could not modify file because it does not exist: ${FILE_TO_MODIFY} "
+    echo " "
+    echo " "
+fi
+
+echo "${ME}: INFO: Fixed openshift scripts."
+

jenkins/master/install_jenkins_lts.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+set -eu -o pipefail
+
+ME="$(basename $0)"
+echo "${ME}: Upgrading Jenkins to latest LTS version available..."
+
+# sudo wget -O /etc/yum.repos.d/jenkins.repo https://pkg.jenkins.io/redhat-stable/jenkins.repo
+# sudo rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key
+# sudo yum upgrade
+# Add required dependencies for the jenkins package
+# sudo yum install java-11-openjdk
+# sudo yum install jenkins
+# sudo systemctl daemon-reload
+
+DEFAULT_TARGET="/usr/lib/jenkins/jenkins.war"
+TARGET="${DEFAULT_TARGET}"
+
+curl -sSLO https://get.jenkins.io/war-stable/latest/jenkins.war
+
+if [ ! -f "${TARGET}" ]; then
+    echo "${ME}: File does not exist: ${TARGET}"
+    TARGET="$(find /usr/ -name jenkins.war)"
+    echo "${ME}: New target: ${TARGET}"
+fi
+
+if [ -f "${TARGET}" ]; then
+    echo "${ME}: Upgrading Jenkins to latest LTS version... "
+    rm -fv ${TARGET}
+    mv -vf jenkins.war ${TARGET}
+    ls -lah ${TARGET}
+else
+    echo "${ME}: ERROR: Cannot upgrade Jenkins version."
+    exit 1
+fi
+
+if [ ! -f "${DEFAULT_TARGET}" ]; then
+    DEFAULT_TARGET_FOLDER="$(dirname ${DEFAULT_TARGET})"
+    if [ ! -d ${DEFAULT_TARGET_FOLDER} ]; then
+        mkdir -pv ${DEFAULT_TARGET_FOLDER}
+    fi
+    cd ${DEFAULT_TARGET_FOLDER} && ln -sv ${TARGET} .
+fi
+ls -la ${DEFAULT_TARGET} ${TARGET}
+echo "${ME}: INFO: Jenkins was upgraded to latest LTS version."
+