From 1772e8f2cfd52a4ffd8d4c2bff3ac016f70ddf4b Mon Sep 17 00:00:00 2001 From: Hesham Elbashandy Date: Thu, 9 Jan 2020 11:56:34 -0800 Subject: [PATCH 1/3] Issue #134: Pass a flag to upgrade manually --- docker-entrypoint.sh | 84 +++++++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 33 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 3407057..968785b 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -37,6 +37,8 @@ if [ "$1" = 'bin/catalina.sh' ]; then unzip $METACAT_WAR -d $METACAT_DIR fi + + # change the context in the web.xml file apply_context.py metacat ${METACAT_APP_CONTEXT} @@ -239,7 +241,6 @@ if [ "$1" = 'bin/catalina.sh' ]; then sleep 10 - # Login to Metacat Admin and start a session (cookie.txt) echo echo '**************************************' @@ -248,10 +249,12 @@ if [ "$1" = 'bin/catalina.sh' ]; then echo '**************************************' echo + # Login to Metacat Admin and start a session (cookie.txt) curl -v -X POST \ --data "loginAction=Login&configureType=login&processForm=true&password=${ADMINPASS}&username=${ADMIN}" \ --cookie-jar /tmp/cookie.txt http://localhost:8080/${METACAT_APP_CONTEXT}/admin > /tmp/login_result.txt 2>&1 + # Test the the admin logged in successfully [ -f /tmp/login_result.txt ] && [ $(grep "User logged in as:" /tmp/login_result.txt| wc -l) -eq 1 ] || (echo "Administrator not logged in!!" && [ ! $DEBUG -eq 1 ] && grep "" /tmp/login_result.txt && exit -4) @@ -261,52 +264,67 @@ if [ "$1" = 'bin/catalina.sh' ]; then echo '**************************************' echo - ## If the DB needs to be updated run the migration scripts - DB_CONFIGURED=`grep "configureType=database" /tmp/login_result.txt | wc -l` - if [ $DB_CONFIGURED -ne 0 ]; - then + if [ "${ESSDIVE_METACAT_MANUAL_UPGRADE}" != 1 ] ; then - # Run the database initialization to create or upgrade tables - # /${METACAT_APP_CONTEXT}/admin?configureType=database must have an authenticated session, then run - curl -X POST --cookie /tmp/cookie.txt \ - --data "configureType=database&processForm=true" \ - http://localhost:8080/${METACAT_APP_CONTEXT}/admin > /dev/null 2>&1 + ## If the DB needs to be updated run the migration scripts + DB_CONFIGURED=`grep "configureType=database" /tmp/login_result.txt | wc -l` + if [ $DB_CONFIGURED -ne 0 ]; + then - # Validate the database should be configured - curl -X POST --cookie /tmp/cookie.txt \ - --data "configureType=configure&processForm=false" \ - http://localhost:8080/${METACAT_APP_CONTEXT}/admin > /dev/null 2>&1 + # Run the database initialization to create or upgrade tables + # /${METACAT_APP_CONTEXT}/admin?configureType=database must have an authenticated session, then run + curl -X POST --cookie /tmp/cookie.txt \ + --data "configureType=database&processForm=true" \ + http://localhost:8080/${METACAT_APP_CONTEXT}/admin > /dev/null 2>&1 - sleep 10 + # Validate the database should be configured + curl -X POST --cookie /tmp/cookie.txt \ + --data "configureType=configure&processForm=false" \ + http://localhost:8080/${METACAT_APP_CONTEXT}/admin > /dev/null 2>&1 - # stop tomcat and ignore exit signal - /bin/catalina.sh stop > /dev/null 2>&1 || true + sleep 10 + # stop tomcat and ignore exit signal + /bin/catalina.sh stop > /dev/null 2>&1 || true - # Give time for tomcat to stop - echo - echo '**************************************' - echo "Waiting for Tomcat to stop before" - echo "restarting after upgrade/initialization" - echo '**************************************' - echo - sleep 10 + + # Give time for tomcat to stop + echo + echo '**************************************' + echo "Waiting for Tomcat to stop before" + echo "restarting after upgrade/initialization" + echo '**************************************' + echo + sleep 10 - # Start tomcat - $@ > /dev/null 2>&1 + # Start tomcat + $@ > /dev/null 2>&1 + + echo + echo '***********************************' + echo "Upgraded/Initialized the metacat DB" + echo '***********************************' + echo + else + echo + echo '**************************************' + echo "Metacat is already configured" + echo '**************************************' + echo + fi - echo - echo '***********************************' - echo "Upgraded/Initialized the metacat DB" - echo '***********************************' - echo else + echo echo '**************************************' - echo "Metacat is already configured" + echo '**************************************' + echo "Metacat is configured to be upgraded manually" + echo "Please go to http://localhost:8080/${METACAT_APP_CONTEXT}/admin to upgrade metacat" + echo '**************************************' echo '**************************************' echo + fi # Remove the session cookie From 5a2a50371f9c2376da51cdedcb5498abf833e492 Mon Sep 17 00:00:00 2001 From: Hesham Elbashandy Date: Wed, 22 Jan 2020 21:45:48 -0800 Subject: [PATCH 2/3] Adds upgrade check Issue ess-dive/ess-dive-project#134: - Check upgrade status from metacat node capabilities - Make sure port 8080 is avaialble before starting tomcat again --- Dockerfile | 5 ++- docker-entrypoint.sh | 105 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 99 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 16179b1..ec4795e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,11 +19,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ python-bcrypt \ vim \ netcat \ + libxml2-utils \ + net-tools \ + telnetd \ + procps \ && rm -rf /var/lib/apt/lists/* \ && cp /tmp/metacat.war /tmp/metacat-index.war /usr/local/tomcat/webapps \ && cat /tmp/catalina.properties >> /usr/local/tomcat/conf/catalina.properties - COPY apply_config.py /usr/local/bin/ RUN ln -s usr/local/bin/apply_config.py / # backwards compat diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 968785b..7afbf13 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -15,6 +15,10 @@ if [ "$1" = 'bin/catalina.sh' ]; then METACAT_WAR=${METACAT_DIR}.war METACATUI_BASE_SKIN_PATH=/usr/local/tomcat/webapps/catalog/style/skins/metacatui + # The following constant defines the last version before the metacat version that supports the upgrade status + # ability in metacat node capabilities. This is used to enable the script to learn when to use the node capability + UPGRADE_STATUS_ABILITY_PRE_VERSION=2.12.2 + # Expand the metacat-index.war if [ ! -d webapps/metacat-index ]; then @@ -37,22 +41,19 @@ if [ "$1" = 'bin/catalina.sh' ]; then unzip $METACAT_WAR -d $METACAT_DIR fi - - # change the context in the web.xml file apply_context.py metacat ${METACAT_APP_CONTEXT} DEFAULT_PROPERTIES_FILE=${METACAT_DIR}/WEB-INF/metacat.properties APP_PROPERTIES_FILE=${APP_PROPERTIES_FILE:-/config/app.properties} METACATUI_CUSTOM_SKINS_PATH=/config/skins + METACAT_VERSION_CONFIGURED=`grep application.metacatVersion $DEFAULT_PROPERTIES_FILE | sed 's/[^:]*=//'` # Look for the metacat ui skin directory if [ -d ${METACATUI_CUSTOM_SKINS_PATH} ]; then - - echo echo '**********************************************************' echo "Synchronizing skins from ${METACATUI_CUSTOM_SKINS_PATH} " @@ -264,7 +265,8 @@ if [ "$1" = 'bin/catalina.sh' ]; then echo '**************************************' echo - if [ "${ESSDIVE_METACAT_MANUAL_UPGRADE}" != 1 ] ; then + + if [ "${MANUAL_UPGRADE}" != 1 ] ; then ## If the DB needs to be updated run the migration scripts DB_CONFIGURED=`grep "configureType=database" /tmp/login_result.txt | wc -l` @@ -282,21 +284,104 @@ if [ "$1" = 'bin/catalina.sh' ]; then --data "configureType=configure&processForm=false" \ http://localhost:8080/${METACAT_APP_CONTEXT}/admin > /dev/null 2>&1 - sleep 10 - # stop tomcat and ignore exit signal - /bin/catalina.sh stop > /dev/null 2>&1 || true + # Check if configured version is 2.12.3 or above as it's the starting version that supports + # upgrade_status field. + if [ "$METACAT_VERSION_CONFIGURED" = "`echo -e "$METACAT_VERSION_CONFIGURED\n$UPGRADE_STATUS_ABILITY_PRE_VERSION" | \ + sort --version-sort | tail -n1`" ] && \ + [ "$METACAT_VERSION_CONFIGURED" != "$UPGRADE_STATUS_ABILITY_PRE_VERSION" ]; then + + # Wait until the upgrade is done. + NEXT_WAIT_TIME=0 + UPGRADED=0 + until [ $UPGRADED -eq 1 ]; do + + sleep $(( NEXT_WAIT_TIME++ )) + + # Wait for total of 5 minutes (Sigma 24) seconds before timeout and + # increase wait time every time gradually. + if [ $NEXT_WAIT_TIME -eq 24 ]; then + echo "****************************************************************************" + echo "******************************** ERROR *************************************" + echo "ERROR: The metacat upgrade is taking too long." + echo " You may try to pass the MANUAL_UPGRADE variable in the docker-compose with 1 to upgrade " + echo " metacat manually" + echo "****************************************************************************" + echo "****************************************************************************" + exit -1 + fi + + # Get Node capabilities from metacat + NODE_CAPABILITIES_XML=`curl --insecure -X GET http://localhost:8080/catalog/d1/mn/v2/node` + UPGRADE_STATUS=`xmllint --xpath '//property[@key="upgrade_status"]/text()' - <<< $NODE_CAPABILITIES_XML` + CURRENT_METACAT_VERSION=`xmllint --xpath '//property[@key="metacat_version"]/text()' - <<< $NODE_CAPABILITIES_XML` + + if [ "$UPGRADE_STATUS" = "success" ]; then + echo "***************************************************************************" + echo "************************* Upgrade successful ******************************" + echo "Metacat now finished the upgrade." + echo " Metacat upgraded version is ${CURRENT_METACAT_VERSION} " + echo "****************************************************************************" + echo "****************************************************************************" + UPGRADED=1 + fi + done + + else + + # if this is an older version than 2.12.3, sleep for 10 seconds until the upgrade finishes + sleep 10 + + fi + + # Saving tomcat PID in a file to be read by the shutdown script + echo `ps h -C java -o "%p:%a" | grep catalina | cut -d: -f1` > /usr/local/tomcat/logs/tomcat.pid + + # Setting the path of the tomcat process id file in a tomcat's special environment variable + export CATALINA_PID=/usr/local/tomcat/logs/tomcat.pid - # Give time for tomcat to stop echo echo '**************************************' echo "Waiting for Tomcat to stop before" echo "restarting after upgrade/initialization" echo '**************************************' echo - sleep 10 + # Shutdown tomcat background process taking hold of port 8080 and ignore exit signal + # Note: after the shutdown script runs it removes the created file above (tomcat.pid) as it would be + # deprecated. + ./bin/catalina.sh stop 30 -force > /dev/null 2>&1 || true + + + echo + echo '**************************************' + echo "Waiting for port 8080 to be available..." + echo '**************************************' + + # Set to 60 to timeout after 1 minute of waiting. + TIMEOUT=600 + while nc -z localhost 8080; do + sleep 0.1 + (( TIMEOUT-- )) + if [ $TIMEOUT -eq 0 ]; then + echo "****************************************************************************" + echo "******************************** WARNING ***********************************" + echo "WARNING: entrypoint script waited for more than 60 seconds but port 8080 was" + echo " still not available." + echo "*** STARTING TOMCAT ANYWAY ***" + echo "****************************************************************************" + echo "****************************************************************************" + break + fi + done + + if [ $TIMEOUT -ne 0 ]; then + echo '**************************************' + echo "Port 8080 is now available" + echo "Proceeding to starting tomcat" + echo '**************************************' + fi # Start tomcat $@ > /dev/null 2>&1 From abdf3f04662eccab565586fa0d27837e8d6e98ad Mon Sep 17 00:00:00 2001 From: Val Hendrix Date: Wed, 5 Feb 2020 18:58:24 -0800 Subject: [PATCH 3/3] Refactor upgrade logic to exit Closes ess-dive/ess-dive-project#134 The upgrade requires a restart and we can't confidently shutdown tomcat before a restart. The best course of action is to exit the container after an upgrade which forces the container manager to start up a new container with the upgaded application. Logic for upgrade + Perform upgrade + Determine it completes successfully + shutdown tomcat + wait for application ports to be release + exit container --- docker-entrypoint.sh | 55 ++++++++++++++------------------------------ 1 file changed, 17 insertions(+), 38 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 7afbf13..7bc2b51 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -239,7 +239,9 @@ if [ "$1" = 'bin/catalina.sh' ]; then echo "checking upgrade/initialization status" echo '**************************************' echo - sleep 10 + while ! nc -z localhost 8080; do + sleep 0.1 + done # Login to Metacat Admin and start a session (cookie.txt) @@ -335,62 +337,39 @@ if [ "$1" = 'bin/catalina.sh' ]; then fi - # Saving tomcat PID in a file to be read by the shutdown script - echo `ps h -C java -o "%p:%a" | grep catalina | cut -d: -f1` > /usr/local/tomcat/logs/tomcat.pid - - # Setting the path of the tomcat process id file in a tomcat's special environment variable - export CATALINA_PID=/usr/local/tomcat/logs/tomcat.pid echo echo '**************************************' - echo "Waiting for Tomcat to stop before" - echo "restarting after upgrade/initialization" + echo "Waiting for Tomcat to stop " + echo "after upgrade/initialization" echo '**************************************' echo - # Shutdown tomcat background process taking hold of port 8080 and ignore exit signal - # Note: after the shutdown script runs it removes the created file above (tomcat.pid) as it would be - # deprecated. - ./bin/catalina.sh stop 30 -force > /dev/null 2>&1 || true - + # Shutdown tomcat background process taking hold of port 8080 and put it in the background + # Saving tomcat PID in a file to be read by the shutdown script + echo `ps h -C java -o "%p:%a" | grep catalina | cut -d: -f1` > /tmp/tomcat.pid + CATALINA_PID=/tmp/tomcat.pid ./bin/catalina.sh stop 120 -force > /dev/null 2>&1 & echo echo '**************************************' - echo "Waiting for port 8080 to be available..." + echo "Waiting for ports 8080,8009,5701 to be released..." echo '**************************************' - # Set to 60 to timeout after 1 minute of waiting. - TIMEOUT=600 - while nc -z localhost 8080; do + # Wait for ports to be released (metacat, hazelcast and apr ) + # NOTE: This does not perfectly tell us that tomcat is down but + # it is our best guess. The catalina.sh script does not stop the + # tomcat process which makes it difficult to determine when tomcat is down. + while nc -z localhost 8080 || nc -z localhost 8009 || nc -z localhost 5701; do sleep 0.1 - (( TIMEOUT-- )) - if [ $TIMEOUT -eq 0 ]; then - echo "****************************************************************************" - echo "******************************** WARNING ***********************************" - echo "WARNING: entrypoint script waited for more than 60 seconds but port 8080 was" - echo " still not available." - echo "*** STARTING TOMCAT ANYWAY ***" - echo "****************************************************************************" - echo "****************************************************************************" - break - fi done - if [ $TIMEOUT -ne 0 ]; then - echo '**************************************' - echo "Port 8080 is now available" - echo "Proceeding to starting tomcat" - echo '**************************************' - fi - - # Start tomcat - $@ > /dev/null 2>&1 - echo echo '***********************************' echo "Upgraded/Initialized the metacat DB" + echo " RESTART the CONTAINER " echo '***********************************' echo + exit else echo echo '**************************************'