diff --git a/jenkins/agent-base/Dockerfile.centos7 b/jenkins/agent-base/Dockerfile.centos7 index 3394c3bff..9815cc246 100644 --- a/jenkins/agent-base/Dockerfile.centos7 +++ b/jenkins/agent-base/Dockerfile.centos7 @@ -11,14 +11,12 @@ ENV SONAR_SCANNER_VERSION=3.1.0.1141 \ GIT_LFS_VERSION=2.6.1 \ SKOPEO_VERSION=0.1.37-3 \ OSTREE_VERSION=2018.5-1 \ - JAVA_TOOL_OPTIONS="-XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Dsun.zip.disableMemoryMapping=true" + JAVA_TOOL_OPTIONS="-XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:-UseContainerSupport -Dsun.zip.disableMemoryMapping=true" ARG APP_DNS ARG SNYK_DISTRIBUTION_URL ARG AQUASEC_SCANNERCLI_URL -ENV JAVA_HOME=/usr/lib/jvm/jre - RUN rm -fv /etc/yum.repos.d/CentOS-Media.repo /etc/yum.repos.d/origin-local-release.repo \ && yum -y install openssl \ && yum -y install java-11-openjdk-devel \ @@ -38,9 +36,9 @@ RUN chmod +x /usr/local/bin/use-j*.sh && \ use-j11.sh && \ echo "--- ENDS JDK 11 TESTS ---" - COPY ./import_certs.sh /usr/local/bin/import_certs.sh -RUN import_certs.sh +COPY ./fix_java_certs_permissions.sh /usr/local/bin/fix_java_certs_permissions.sh +RUN import_certs.sh && fix_java_certs_permissions.sh # Install Sonar Scanner. RUN cd /tmp \ diff --git a/jenkins/agent-base/Dockerfile.ubi8 b/jenkins/agent-base/Dockerfile.ubi8 index bfb5a4c2b..d9b15b3df 100644 --- a/jenkins/agent-base/Dockerfile.ubi8 +++ b/jenkins/agent-base/Dockerfile.ubi8 @@ -9,7 +9,7 @@ ENV SONAR_SCANNER_VERSION=3.1.0.1141 \ HELM_PLUGIN_DIFF_VERSION=3.3.2 \ HELM_PLUGIN_SECRETS_VERSION=3.3.5 \ GIT_LFS_VERSION=2.6.1 \ - JAVA_TOOL_OPTIONS="-XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Dsun.zip.disableMemoryMapping=true" + JAVA_TOOL_OPTIONS="-XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:-UseContainerSupport -Dsun.zip.disableMemoryMapping=true" ARG APP_DNS ARG SNYK_DISTRIBUTION_URL @@ -39,7 +39,8 @@ RUN chmod +x /usr/local/bin/use-j*.sh && \ echo "--- ENDS JDK 11 TESTS ---" COPY ./import_certs.sh /usr/local/bin/import_certs.sh -RUN import_certs.sh +COPY ./fix_java_certs_permissions.sh /usr/local/bin/fix_java_certs_permissions.sh +RUN import_certs.sh && fix_java_certs_permissions.sh # Install Sonar Scanner. RUN cd /tmp \ diff --git a/jenkins/agent-base/fix_java_certs_permissions.sh b/jenkins/agent-base/fix_java_certs_permissions.sh new file mode 100755 index 000000000..5ec50efe6 --- /dev/null +++ b/jenkins/agent-base/fix_java_certs_permissions.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -eu + +# Initialize JAVA_HOME if not set. +JAVA_HOME=${JAVA_HOME:-""} + +if [ -f /etc/profile.d/set-default-java.sh ]; then + source /etc/profile.d/set-default-java.sh +else + echo "WARNING: Not setting default java version." +fi + +echo "Trying to setup correct permissions for cacerts folder... " +if [ ! -z "${JAVA_HOME}" ] && [ "" != "${JAVA_HOME}" ]; then + chown -c 1001:0 $JAVA_HOME/lib/security/cacerts + chmod -c g+w $JAVA_HOME/lib/security/cacerts +else + echo "WARNING: Cannot apply permissions 'chmod g+w' to JAVA_HOME/lib/security/cacerts " + echo "WARNING: JAVA_HOME=${JAVA_HOME}" +fi diff --git a/jenkins/agent-base/import_certs.sh b/jenkins/agent-base/import_certs.sh index 56bd718f9..bcf771e75 100755 --- a/jenkins/agent-base/import_certs.sh +++ b/jenkins/agent-base/import_certs.sh @@ -32,12 +32,3 @@ if [[ ! -z ${APP_DNS:=""} ]]; then else echo 'No certificates to import' fi - -echo "Trying to setup correct permissions for cacerts folder... " -if [ ! -z "${JAVA_HOME}" ] && [ "" != "${JAVA_HOME}" ]; then - chown -c 1001:0 $JAVA_HOME/lib/security/cacerts - chmod -c g+w $JAVA_HOME/lib/security/cacerts -else - echo "WARNING: Cannot apply permissions 'chmod g+w' to JAVA_HOME/lib/security/cacerts " - echo "WARNING: JAVA_HOME=${JAVA_HOME}" -fi diff --git a/jenkins/master/Dockerfile.centos7 b/jenkins/master/Dockerfile.centos7 index 4c298d69e..38cfd048d 100644 --- a/jenkins/master/Dockerfile.centos7 +++ b/jenkins/master/Dockerfile.centos7 @@ -25,6 +25,7 @@ RUN /usr/local/bin/install-plugins.sh /opt/openshift/configuration/plugins.txt \ && mv /usr/libexec/s2i/run /usr/libexec/s2i/openshift-run COPY configuration/ /opt/openshift/configuration/ COPY ods-run.sh /usr/libexec/s2i/run +COPY logging.properties /var/lib/jenkins/ RUN chown :0 /etc/pki/java/cacerts && chmod ugo+w /etc/pki/java/cacerts diff --git a/jenkins/master/Dockerfile.rhel7 b/jenkins/master/Dockerfile.rhel7 index 0fc33575f..1037e17e1 100644 --- a/jenkins/master/Dockerfile.rhel7 +++ b/jenkins/master/Dockerfile.rhel7 @@ -25,6 +25,7 @@ RUN /usr/local/bin/install-plugins.sh /opt/openshift/configuration/plugins.txt \ && mv /usr/libexec/s2i/run /usr/libexec/s2i/openshift-run COPY configuration/ /opt/openshift/configuration/ COPY ods-run.sh /usr/libexec/s2i/run +COPY logging.properties /var/lib/jenkins/ RUN chown :0 /etc/pki/java/cacerts && chmod ugo+w /etc/pki/java/cacerts diff --git a/jenkins/master/Dockerfile.ubi8 b/jenkins/master/Dockerfile.ubi8 index d6531ba23..6e1642577 100644 --- a/jenkins/master/Dockerfile.ubi8 +++ b/jenkins/master/Dockerfile.ubi8 @@ -25,6 +25,7 @@ RUN /usr/local/bin/install-plugins.sh /opt/openshift/configuration/plugins.txt \ && mv /usr/libexec/s2i/run /usr/libexec/s2i/openshift-run COPY configuration/ /opt/openshift/configuration/ COPY ods-run.sh /usr/libexec/s2i/run +COPY logging.properties /var/lib/jenkins/ RUN chown :0 /etc/pki/java/cacerts && chmod ugo+w /etc/pki/java/cacerts diff --git a/jenkins/master/logging.properties b/jenkins/master/logging.properties new file mode 100644 index 000000000..9b0c461c9 --- /dev/null +++ b/jenkins/master/logging.properties @@ -0,0 +1,11 @@ +handlers=java.util.logging.ConsoleHandler,java.util.logging.FileHandler + +java.util.logging.FileHandler.level=INFO +java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.FileHandler.pattern=/var/log/jenkins/jenkins-master.log +java.util.logging.FileHandler.append=true +java.util.logging.FileHandler.limit=10000000 +java.util.logging.FileHandler.count=5 + +java.util.logging.ConsoleHandler.level=INFO +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter diff --git a/jenkins/ocp-config/build/bc.yml b/jenkins/ocp-config/build/bc.yml index 063f1ce3a..99eb4e219 100644 --- a/jenkins/ocp-config/build/bc.yml +++ b/jenkins/ocp-config/build/bc.yml @@ -72,7 +72,7 @@ objects: labels: app: jenkins spec: - failedBuildsHistoryLimit: 5 + failedBuildsHistoryLimit: 20 nodeSelector: null output: to: @@ -117,7 +117,7 @@ objects: from: kind: DockerImage name: ${JENKINS_MASTER_BASE_FROM_IMAGE} - successfulBuildsHistoryLimit: 5 + successfulBuildsHistoryLimit: 20 - kind: BuildConfig apiVersion: v1 metadata: @@ -125,7 +125,7 @@ objects: labels: app: jenkins spec: - failedBuildsHistoryLimit: 5 + failedBuildsHistoryLimit: 20 nodeSelector: null output: to: @@ -164,7 +164,7 @@ objects: name: ${JENKINS_AGENT_BASE_FROM_IMAGE} dockerfilePath: ${JENKINS_AGENT_DOCKERFILE_PATH} type: Docker - successfulBuildsHistoryLimit: 5 + successfulBuildsHistoryLimit: 20 - apiVersion: v1 kind: BuildConfig metadata: diff --git a/jenkins/ocp-config/deploy/jenkins-master.yml b/jenkins/ocp-config/deploy/jenkins-master.yml index cfbfb1029..756cdc7fe 100644 --- a/jenkins/ocp-config/deploy/jenkins-master.yml +++ b/jenkins/ocp-config/deploy/jenkins-master.yml @@ -17,7 +17,7 @@ parameters: - name: JENKINS_ENABLE_OAUTH value: "true" - name: JENKINS_MEMORY_LIMIT - value: 6Gi + value: 7Gi - name: JENKINS_MEMORY_REQUEST value: 4Gi - name: JENKINS_CPU_LIMIT @@ -27,7 +27,7 @@ parameters: - name: JENKINS_VOLUME_CAPACITY value: 5Gi - name: JENKINS_JAVA_GC_OPTS - value: "-server -XX:NativeMemoryTracking=summary -XX:MaxRAMPercentage=90 -XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication -XX:MaxMetaspaceSize=1g -XX:MetaspaceSize=256M -XX:SoftRefLRUPolicyMSPerMB=1 -XX:+UnlockExperimentalVMOptions -XX:+UnlockDiagnosticVMOptions -XX:G1SummarizeRSetStatsPeriod=1" + value: "-server -XX:NativeMemoryTracking=summary -XX:-UseContainerSupport -XX:MaxRAMPercentage=90 -XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication -XX:MaxMetaspaceSize=1g -XX:MetaspaceSize=256M -XX:SoftRefLRUPolicyMSPerMB=1 -XX:+UnlockExperimentalVMOptions -XX:+UnlockDiagnosticVMOptions -XX:G1SummarizeRSetStatsPeriod=1 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/lib/jenkins" - name: JENKINS_JAVA_MAX_HEAP_PARAM value: "-Xms1024m -Xmx4g" - name: JENKINS_CONTAINER_HEAP_PERCENT diff --git a/ods-devenv/scripts/deploy.sh b/ods-devenv/scripts/deploy.sh index 957652ff7..0b4a14f92 100755 --- a/ods-devenv/scripts/deploy.sh +++ b/ods-devenv/scripts/deploy.sh @@ -1574,10 +1574,13 @@ function wait_until_http_svc_is_up() { local SVC_HTTP_URL="${2}" local CURL_SVC_OUTPUT_FILE="/tmp/result-curl-svc-${SVC_NAME}-output" local CURL_SVC_HEADERS_FILE="/tmp/result-curl-svc-${SVC_NAME}-headers" + local CURL_LOGS_CHECK_SVC_FILE="/tmp/result-curl-svc-${SVC_NAME}-curlresult" local retryMax=${3:-20} - wait_until_http_svc_is_up_advanced "$SVC_NAME" "$SVC_HTTP_URL" "$CURL_SVC_OUTPUT_FILE" "$CURL_SVC_HEADERS_FILE" $retryMax - if [ 0 -ne $? ]; then + local RETURN_VALUE=0 + wait_until_http_svc_is_up_advanced "$SVC_NAME" "$SVC_HTTP_URL" "$CURL_SVC_OUTPUT_FILE" "$CURL_SVC_HEADERS_FILE" \ + "${CURL_LOGS_CHECK_SVC_FILE}" $retryMax || RETURN_VALUE=1 + if [ 0 -ne ${RETURN_VALUE} ]; then echo "[STATUS CHECK] ERROR: Service is down and we cannot live without it: ${SVC_NAME}" return 1 fi @@ -1589,7 +1592,8 @@ function wait_until_http_svc_is_up_advanced() { local SVC_HTTP_URL="${2}" local CURL_SVC_OUTPUT_FILE="${3}" local CURL_SVC_HEADERS_FILE="${4}" - local retryMaxIn=${5:-20} + local CURL_LOGS_CHECK_SVC_FILE="${5}" + local retryMaxIn=${6:-20} local retryMax=$((retryMaxIn)) echo " " @@ -1620,14 +1624,18 @@ function wait_until_http_svc_is_up_advanced() { fi # Remove files from previous execution. - rm -fv ${CURL_SVC_OUTPUT_FILE} ${CURL_SVC_HEADERS_FILE} || true + rm -fv ${CURL_SVC_OUTPUT_FILE} ${CURL_SVC_HEADERS_FILE} ${CURL_LOGS_CHECK_SVC_FILE} || true if [ -f ${CURL_SVC_OUTPUT_FILE} ] || [ -f ${CURL_SVC_HEADERS_FILE} ]; then echo "[STATUS CHECK] WARNING: Could NOT remove files ${CURL_SVC_OUTPUT_FILE} ${CURL_SVC_HEADERS_FILE} " fi - if ! curl --insecure -sSL --retry-delay 2 --retry-max-time 20 --retry 10 --dump-header ${CURL_SVC_HEADERS_FILE} ${SVC_HTTP_URL} -o ${CURL_SVC_OUTPUT_FILE} ; then + local CURL_RETURN_VAL=0 + curl --insecure -sSL --retry-delay 2 --retry-max-time 20 --retry 10 --dump-header ${CURL_SVC_HEADERS_FILE} \ + -o ${CURL_SVC_OUTPUT_FILE} ${SVC_HTTP_URL} 2>&1 > ${CURL_LOGS_CHECK_SVC_FILE} || CURL_RETURN_VAL=1 + if [ 0 -ne ${CURL_RETURN_VAL} ]; then echo "Curl replied != 0 for query to ${SVC_HTTP_URL} " echo "Checking if it was caused by a redirect... " + grep -i 'HTTP' ${CURL_LOGS_CHECK_SVC_FILE} || true fi if ! grep -q '^\s*HTTP/[0-9\.]*\s*200[\s]*' ${CURL_SVC_HEADERS_FILE} ; then @@ -2703,9 +2711,12 @@ function check_pods_and_restart_if_necessary() { local retryMaxIn=${1:-5} local retryMaxHttpIn=${2:-10} - check_pod_and_restart_if_necessary 'sonarqube' 'ods/sonarqube' 'https://sonarqube-ods.ocp.odsbox.lan/' ${retryMaxIn} ${retryMaxHttpIn} - check_pod_and_restart_if_necessary 'prov-app' 'ods/ods-provisioning-app' 'https://prov-app-ods.ocp.odsbox.lan/' ${retryMaxIn} ${retryMaxHttpIn} - check_pod_and_restart_if_necessary 'nexus' 'ods/nexus' 'https://nexus-ods.ocp.odsbox.lan/' ${retryMaxIn} ${retryMaxHttpIn} + check_pod_and_restart_if_necessary 'sonarqube' 'ods/sonarqube' 'https://sonarqube-ods.ocp.odsbox.lan/' \ + ${retryMaxIn} ${retryMaxHttpIn} || restart_ods + check_pod_and_restart_if_necessary 'prov-app' 'ods/ods-provisioning-app' 'https://prov-app-ods.ocp.odsbox.lan/' \ + ${retryMaxIn} ${retryMaxHttpIn} || restart_ods + check_pod_and_restart_if_necessary 'nexus' 'ods/nexus' 'https://nexus-ods.ocp.odsbox.lan/' \ + ${retryMaxIn} ${retryMaxHttpIn} || restart_ods # https://jenkins-ods.ocp.odsbox.lan } @@ -2716,6 +2727,7 @@ function check_pod_and_restart_if_necessary() { local SVC_HTTP_URL="${3}" local CURL_SVC_OUTPUT_FILE="/tmp/result-curl-svc-${SVC_NAME}-output" local CURL_SVC_HEADERS_FILE="/tmp/result-curl-svc-${SVC_NAME}-headers" + local CURL_LOGS_CHECK_SVC_FILE="/tmp/result-curl-svc-${SVC_NAME}-curlresult" local retryMaxIn=${4:-5} local retryMax=$((retryMaxIn)) local retryMaxHttpIn=${5:-10} @@ -2735,7 +2747,8 @@ function check_pod_and_restart_if_necessary() { fi retVal=0 - wait_until_http_svc_is_up_advanced "$SVC_NAME" "$SVC_HTTP_URL" "$CURL_SVC_OUTPUT_FILE" "$CURL_SVC_HEADERS_FILE" ${retryMaxHttpIn} || retVal=1 + wait_until_http_svc_is_up_advanced "$SVC_NAME" "$SVC_HTTP_URL" "$CURL_SVC_OUTPUT_FILE" "$CURL_SVC_HEADERS_FILE" \ + ${CURL_LOGS_CHECK_SVC_FILE} ${retryMaxHttpIn} || retVal=1 if [ 0 -ne ${retVal} ]; then echo "[STATUS CHECK] WARNING: Stopping pod so it restarts automatically. Service: ${SVC_NAME} " @@ -2749,7 +2762,10 @@ function check_pod_and_restart_if_necessary() { if [ "false" == "$docker_process_killed" ]; then echo "No docker process found for pod ${SVC_NAME} with ID ${SVC_POD_ID} " + echo "Current docker pods: " + docker ps -a | grep -v 'Exited .* ago' || true echo " " + return 1 fi fi diff --git a/tests/scripts/free-unused-resources.sh b/tests/scripts/free-unused-resources.sh index 4fdd24ac9..83890c00c 100755 --- a/tests/scripts/free-unused-resources.sh +++ b/tests/scripts/free-unused-resources.sh @@ -2,6 +2,8 @@ echo " " +ME=$(basename $0) + function clean_containers { echo "Removing docker containers no more used... " if docker ps -a | grep -q 'Exited .* ago' ; then @@ -27,6 +29,7 @@ function clean_tests { } function clean_odsverify { + echo "Cleaning projects ODS__VERIFY... " if [ "true" == "$CLEAN_ODS_VERIFY" ]; then echo "Removing ODS VERIFY projects..." oc projects | grep '^\s*odsverify.*' | while read -r line; do @@ -41,14 +44,14 @@ function clean_odsverify { } function clean_images { + echo "Cleaning OC images" echo "oc adm prune images --keep-tag-revisions=1 --keep-younger-than=30m --confirm" oc adm prune images --keep-tag-revisions=1 --keep-younger-than=30m --confirm || true } function usage { - ME=$(basename $0) echo " " - echo "usage: ${ME} [--odsVerify] [--omitTestsProject tes22]" + echo "usage: ${ME} [--odsVerify] [--omitTests] [--omitTestsProject tes22]" echo " " } @@ -59,6 +62,7 @@ function echo_error() { OMIT_TESTS_PROJECT=none CLEAN_ODS_VERIFY="false" +CLEAN_TESTS="false" while [[ "$#" -gt 0 ]]; do case $1 in @@ -71,11 +75,19 @@ while [[ "$#" -gt 0 ]]; do --omitTestsProject) OMIT_TESTS_PROJECT="$2"; echo "Tests to omit: $OMIT_TESTS_PROJECT"; shift;; + --cleanTests) CLEAN_TESTS="true";; + *) echo_error "Unknown parameter passed: $1"; exit 1;; esac; shift; done clean_containers -clean_tests +if [ "true" == "${CLEAN_TESTS}" ]; then + clean_tests +else + echo " " + echo "${ME}: INFO: Not cleaning tests" + echo " " +fi clean_odsverify clean_images