Remove code for ReplicaSet and add StatefulSet support for 2.6 and 3.…

…0-upg versions. - Add OpenShift replication tests too - Update decription of run-mongod-pet scripts (remove notes about run-mongod-replication) Remove unused function from common.sh file. Hack to fix having Persistent volume with content from previous run. Rename scripts not to use PetSet related names.
sclorg · Mar 16, 2017 · c8af6e9 · c8af6e9
1 parent eccac30
commit c8af6e9
Show file tree

Hide file tree

Showing 18 changed files with 288 additions and 725 deletions.
diff --git a/2.6/root/usr/bin/run-mongod-pet b/2.6/root/usr/bin/run-mongod-pet
@@ -0,0 +1 @@
+run-mongod-replication
diff --git a/2.6/root/usr/bin/run-mongod-replication b/2.6/root/usr/bin/run-mongod-replication
@@ -1,9 +1,15 @@
 #!/bin/bash
 #
-# Run mongod which connect to replSet
-# $1 - "initiate" (optional) -> this mongod will initiate a replSet
+# Run mongod in a StatefulSet-based replica set. See
+# https://github.com/sclorg/mongodb-container/blob/master/examples/petset/README.md
+# for a description of how this is intended to work.
 #
-# If this script is terminated it removes mongod from replSet
+# Note:
+# - It does not attempt to remove the host from the replica set configuration
+#   when it is terminating. That is by design, because, in a StatefulSet, when a
+#   pod/container terminates and is restarted by OpenShift, it will always have
+#   the same hostname. Removing hosts from the configuration affects replica set
+#   elections and can impact the replica set stability.
 
 set -o errexit
 set -o nounset
@@ -21,7 +27,6 @@ function usage() {
   echo "  MONGODB_REPLICA_NAME"
   echo "Optional variables:"
   echo "  MONGODB_SERVICE_NAME (default: mongodb)"
-  echo "  MONGODB_INITIAL_REPLICA_COUNT"
   echo "MongoDB settings:"
   echo "  MONGODB_NOPREALLOC (default: true)"
   echo "  MONGODB_SMALLFILES (default: true)"
@@ -30,36 +35,6 @@ function usage() {
 }
 
 function cleanup() {
-  if [ "$(mongo admin -u admin -p ${MONGODB_ADMIN_PASSWORD} --quiet --eval 'rs.isMaster().ismaster')" == "true" ]; then
-
-    # Wait that some SECONDARY is synced with PRIMARY
-    echo "=> Waiting for syncing SECONDARY ..."
-    replset_wait_sync
-
-    # Some commands will force MongoDB client to re-connect. This is not working
-    # well in combination with '--eval'. In that case the 'mongo' command will fail
-    # with return code 254.
-    echo "=> Giving up the PRIMARY role ..."
-    mongo admin -u admin -p "${MONGODB_ADMIN_PASSWORD}" --quiet --eval "rs.stepDown(120);" &>/dev/null || true
-
-    # Wait till the new PRIMARY member is elected
-    echo "=> Waiting for the new PRIMARY to be elected ..."
-    mongo admin -u admin -p "${MONGODB_ADMIN_PASSWORD}" --quiet \
-     --eval "var i = ${MAX_ATTEMPTS};
-      while(i > 0) {
-        var members=rs.status().members;
-        for(i=0;i<members.length;i++){
-          if(members[i].stateStr=='PRIMARY' && members[i].name!='$(mongo_addr)'){
-            quit(0)
-          }
-        };
-        sleep(${SLEEP_TIME}*1000)
-      };" &>/dev/null
-    echo "=> A new PRIMARY member was elected, removing from replset ..."
-  fi
-  if [ -n "${MONGODB_REPLICA_NAME-}" ]; then
-    mongo_remove
-  fi
   echo "=> Shutting down MongoDB server ..."
   pkill -INT mongod || :
   wait_for_mongo_down
@@ -74,32 +49,23 @@ if [ ! -s "${MONGODB_CONFIG_PATH}" ]; then
   envsubst < "${CONTAINER_SCRIPTS_PATH}/mongodb.conf.template" > "${MONGODB_CONFIG_PATH}"
 fi
 
-# Need to cache the container address for the cleanup
-cache_container_addr
-mongo_common_args="-f $MONGODB_CONFIG_PATH"
 if ! [[ -v MONGODB_USER && -v MONGODB_PASSWORD && -v MONGODB_DATABASE && -v MONGODB_ADMIN_PASSWORD && -v MONGODB_KEYFILE_VALUE && -v MONGODB_REPLICA_NAME ]]; then
   usage
 fi
 
-# Run the MongoDB in 'replicated' mode
+mongo_common_args="-f ${MONGODB_CONFIG_PATH}"
+
+# Attention: setup_keyfile may modify value of mongo_common_args!
 setup_keyfile
 
-# Initialize the replica set or add member in the background.
-${CONTAINER_SCRIPTS_PATH}/init-replset.sh "${1:-}" "$$" &
+${CONTAINER_SCRIPTS_PATH}/init-replset.sh &
 
-# TODO: capture exit code of `init-replset.sh` and exit with an error if the
-# initialization failed, so that the container will be restarted and the user
-# can gain more visibility that there is a problem in a way other than just
+# TODO: capture exit code of `init-petset-replset.sh` and exit with an error if
+# the initialization failed, so that the container will be restarted and the
+# user can gain more visibility that there is a problem in a way other than just
 # inspecting log messages.
 
-# Don't pass signals to background processes. Background processes are killed in `cleanup`.
-set -m
-# Run `mongod` in a subshell because MONGODB_ADMIN_PASSWORD should still be
-# defined when the trapped call to `cleanup` references it.
-(
-  # Make sure env variables don't propagate to mongod process.
-  unset MONGODB_USER MONGODB_PASSWORD MONGODB_DATABASE MONGODB_ADMIN_PASSWORD
-
-  mongod $mongo_common_args --replSet "${MONGODB_REPLICA_NAME}"
-) &
+# Make sure env variables don't propagate to mongod process.
+unset MONGODB_USER MONGODB_PASSWORD MONGODB_DATABASE MONGODB_ADMIN_PASSWORD
+mongod ${mongo_common_args} --replSet "${MONGODB_REPLICA_NAME}" &
 wait
diff --git a/2.6/root/usr/share/container-scripts/mongodb/common.sh b/2.6/root/usr/share/container-scripts/mongodb/common.sh
@@ -21,31 +21,6 @@ MONGODB_KEYFILE_PATH="${HOME}/keyfile"
 readonly MAX_ATTEMPTS=60
 readonly SLEEP_TIME=1
 
-# container_addr returns the current container external IP address
-function container_addr() {
-  echo -n $(cat ${HOME}/.address)
-}
-
-# mongo_addr returns the IP:PORT of the currently running MongoDB instance
-function mongo_addr() {
-  echo -n "$(container_addr):${CONTAINER_PORT}"
-}
-
-# cache_container_addr waits till the container gets the external IP address and
-# cache it to disk
-function cache_container_addr() {
-  echo -n "=> Waiting for container IP address ..."
-  local i
-  for i in $(seq "$MAX_ATTEMPTS"); do
-    if ip -oneline -4 addr show up scope global | grep -Eo '[0-9]{,3}(\.[0-9]{,3}){3}' > "${HOME}"/.address; then
-      echo " $(mongo_addr)"
-      return 0
-    fi
-    sleep $SLEEP_TIME
-  done
-  echo >&2 "Failed to get Docker container IP address." && exit 1
-}
-
 # wait_for_mongo_up waits until the mongo server accepts incomming connections
 function wait_for_mongo_up() {
   _wait_for_mongo 1 "$@"
@@ -89,28 +64,6 @@ function endpoints() {
   dig ${service_name} A +search +short 2>/dev/null
 }
 
-# replset_config_members builds part of the MongoDB replicaSet config: "members: [...]"
-# used for the cluster initialization.
-# Takes a list of space-separated member IPs as the first argument.
-function replset_config_members() {
-  local current_endpoints
-  current_endpoints="$1"
-  local members
-  members="{ _id: 0, host: \"$(mongo_addr)\"},"
-  local member_id
-  member_id=1
-  local container_addr
-  container_addr="$(container_addr)"
-  local node
-  for node in ${current_endpoints}; do
-    if [ "$node" != "$container_addr" ]; then
-      members+="{ _id: ${member_id}, host: \"${node}:${CONTAINER_PORT}\"},"
-      let member_id++
-    fi
-  done
-  echo -n "members: [ ${members%,} ]"
-}
-
 # replset_addr return the address of the current replSet
 function replset_addr() {
   local current_endpoints
@@ -122,49 +75,6 @@ function replset_addr() {
   echo "${MONGODB_REPLICA_NAME}/${current_endpoints//[[:space:]]/,}"
 }
 
-# replse_wait_sync wait for at least two members to be up to date (PRIMARY and one SECONDARY)
-function replset_wait_sync() {
-  local host
-  # if we cannot determine the IP address of the primary, exit without an error
-  # to allow callers to proceed with their logic
-  host="$(replset_addr || true)"
-  if [ -z "$host" ]; then
-    return 1
-  fi
-
-  mongo admin -u admin -p "${MONGODB_ADMIN_PASSWORD}" --host ${host} \
-    --eval "var i = ${MAX_ATTEMPTS};
-    while(i > 0) {
-      var status=rs.status();
-      var primary_optime=status.members.filter(function(el) {return el.state ==1})[0].optime;
-      // Check that at least one member has same optime as PRIMARY (PRIMARY and one SECONDARY ~ >= 2)
-      if(status.members.filter(function(el) {return tojson(el.optime) == tojson(primary_optime)}).length >= 2)
-        quit(0);
-      else
-        sleep(${SLEEP_TIME}*1000);
-      i--;
-    };
-    quit(1);"
-}
-
-# mongo_remove removes the current MongoDB from the cluster
-function mongo_remove() {
-  local host
-  # if we cannot determine the IP address of the primary, exit without an error
-  # to allow callers to proceed with their logic
-  host="$(replset_addr || true)"
-  if [ -z "$host" ]; then
-    return
-  fi
-
-  local mongo_addr
-  mongo_addr="$(mongo_addr)"
-
-  echo "=> Removing ${mongo_addr} from replica set ..."
-  mongo admin -u admin -p "${MONGODB_ADMIN_PASSWORD}" \
-    --host "${host}" --eval "rs.remove('${mongo_addr}');" || true
-}
-
 # mongo_create_admin creates the MongoDB admin user with password: MONGODB_ADMIN_PASSWORD
 # $1 - login parameters for mongo (optional)
 # $2 - host where to connect (localhost by default)

diff --git a/2.6/root/usr/share/container-scripts/mongodb/init-replset.sh b/2.6/root/usr/share/container-scripts/mongodb/init-replset.sh
@@ -8,7 +8,19 @@ source "${CONTAINER_SCRIPTS_PATH}/common.sh"
 
 # This is a full hostname that will be added to replica set
 # (for example, "replica-2.mongodb.myproject.svc.cluster.local")
-readonly MEMBER_HOST="$(container_addr)"
+readonly MEMBER_HOST="$(hostname -f)"
+
+# Outputs available endpoints (hostnames) to stdout.
+# This also includes hostname of the current pod.
+#
+# Uses the following global variables:
+# - MONGODB_SERVICE_NAME (optional, defaults to 'mongodb')
+function find_endpoints() {
+  local service_name="${MONGODB_SERVICE_NAME:-mongodb}"
+
+  # Extract host names from lines like this: "10 33 0 mongodb-2.mongodb.myproject.svc.cluster.local."
+  dig "${service_name}" SRV +search +short | cut -d' ' -f4 | rev | cut -c2- | rev
+}
 
 # Initializes the replica set configuration.
 #
@@ -18,36 +30,10 @@ readonly MEMBER_HOST="$(container_addr)"
 # Uses the following global variables:
 # - MONGODB_REPLICA_NAME
 # - MONGODB_ADMIN_PASSWORD
-# - MONGODB_INITIAL_REPLICA_COUNT
 function initiate() {
   local host="$1"
 
-  # Wait for all nodes to be listed in endpoints() and accept connections
-  current_endpoints=$(endpoints)
-  if [ -n "${MONGODB_INITIAL_REPLICA_COUNT:-}" ]; then
-    echo -n "=> Waiting for $MONGODB_INITIAL_REPLICA_COUNT MongoDB endpoints ..."
-    while [[ "$(echo "${current_endpoints}" | wc -l)" -lt ${MONGODB_INITIAL_REPLICA_COUNT} ]]; do
-      sleep 2
-      current_endpoints=$(endpoints)
-    done
-  else
-    echo "Attention: MONGODB_INITIAL_REPLICA_COUNT is not set and it could lead to a improperly configured replica set."
-    echo "To fix this, set MONGODB_INITIAL_REPLICA_COUNT variable to the number of members in the replica set in"
-    echo "the configuration of post deployment hook."
-
-    echo -n "=> Waiting for MongoDB endpoints ..."
-    while [ -z "${current_endpoints}" ]; do
-      sleep 2
-      current_endpoints=$(endpoints)
-    done
-  fi
-  echo "${current_endpoints}"
-  echo "=> Waiting for all endpoints to accept connections..."
-  for node in ${current_endpoints}; do
-    wait_for_mongo_up ${node} &>/dev/null
-  done
-
-  local config="{_id: '${MONGODB_REPLICA_NAME}', $(replset_config_members "${current_endpoints}")}"
+  local config="{_id: '${MONGODB_REPLICA_NAME}', members: [{_id: 0, host: '${host}'}]}"
 
   info "Initiating MongoDB replica using: ${config}"
   mongo --eval "quit(rs.initiate(${config}).ok ? 0 : 1)" --quiet
@@ -68,19 +54,24 @@ function initiate() {
 # - $1: host address[:port]
 #
 # Global variables:
+# - MONGODB_REPLICA_NAME
 # - MONGODB_ADMIN_PASSWORD
 function add_member() {
   local host="$1"
   info "Adding ${host} to replica set ..."
 
-  if [ -z "$(endpoints)" ]; then
+  # TODO: replace this with a call to `replset_addr` from common.sh, once it returns host names.
+  local endpoints
+  endpoints="$(find_endpoints | paste -s -d,)"
+
+  if [ -z "${endpoints}" ]; then
     info "ERROR: couldn't add host to replica set!"
     info "CAUSE: DNS lookup for '${MONGODB_SERVICE_NAME:-mongodb}' returned no results."
     return 1
   fi
 
   local replset_addr
-  replset_addr="$(replset_addr)"
+  replset_addr="${MONGODB_REPLICA_NAME}/${endpoints}"
 
   if ! mongo admin -u admin -p "${MONGODB_ADMIN_PASSWORD}" --host "${replset_addr}" --eval "while (!rs.add('${host}').ok) { sleep(100); }" --quiet; then
     info "ERROR: couldn't add host to replica set!"
@@ -93,22 +84,28 @@ function add_member() {
   info "Successfully joined replica set"
 }
 
-info "Waiting for local MongoDB to accept connections ..."
+info "Waiting for local MongoDB to accept connections  ..."
 wait_for_mongo_up &>/dev/null
 
 if [[ $(mongo --eval 'db.isMaster().setName' --quiet) == "${MONGODB_REPLICA_NAME}" ]]; then
   info "Replica set '${MONGODB_REPLICA_NAME}' already exists, skipping initialization"
+  >/tmp/initialized
   exit 0
 fi
 
+# StatefulSet pods are named with a predictable name, following the pattern:
+#   $(statefulset name)-$(zero-based index)
+# MEMBER_ID is computed by removing the prefix matching "*-", i.e.:
+#  "mongodb-0" -> "0"
+#  "mongodb-1" -> "1"
+#  "mongodb-2" -> "2"
+readonly MEMBER_ID="${HOSTNAME##*-}"
+
 # Initialize replica set only if we're the first member
-if [ "$1" == "initiate" ]; then
-  main_process_id=$2
-  # Initiate replica set
+if [ "${MEMBER_ID}" = '0' ]; then
   initiate "${MEMBER_HOST}"
-
-  # Exit this pod
-  kill ${main_process_id}
 else
   add_member "${MEMBER_HOST}"
 fi
+
+>/tmp/initialized