totem · sukrit007 · Apr 13, 2016 · Apr 11, 2016 · Apr 11, 2016 · Apr 11, 2016
diff --git a/.dockerignore b/.dockerignore
@@ -1,10 +1,11 @@
 .idea
 tests
 docs
-.git
+.git*
 *.md
 .travis.yml
 dev-requirements.txt
 *.iml
 LICENSE
-local-*
+local-*
+docker-compose.yml
diff --git a/Dockerfile b/Dockerfile
@@ -1,54 +1,80 @@
-FROM totem/python-base:2.7-trusty-b3
-
-RUN apt-get update --fix-missing && \
-    apt-get install -y gettext && \
-    apt-get clean && rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/*
-
-##SSH Key for fleet
-RUN mkdir /root/.ssh && \
-    chmod  500 /root/.ssh && \
-    chown -R root:root /root/.ssh
-
-#Confd
-ENV CONFD_VERSION 0.11.0
-RUN curl -L https://github.com/kelseyhightower/confd/releases/download/v$CONFD_VERSION/confd-${CONFD_VERSION}-linux-amd64 -o /usr/local/bin/confd && \
-    chmod 555 /usr/local/bin/confd
-
-#Etcdctl
-ENV ETCDCTL_VERSION v2.2.5
-RUN curl -L https://github.com/coreos/etcd/releases/download/$ETCDCTL_VERSION/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz -o /tmp/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz && \
-    cd /tmp && gzip -dc etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz | tar -xof - && \
-    cp -f /tmp/etcd-$ETCDCTL_VERSION-linux-amd64/etcdctl /usr/local/bin && \
-    rm -rf /tmp/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz && \
-    rm -rf /tmp/etcd-$ETCDCTL_VERSION-linux-amd64
-
-# Supervisor and App dependencies
-RUN pip install supervisor==3.1.2 supervisor-stdout
-ADD requirements.txt /opt/requirements.txt
-RUN pip install -r /opt/requirements.txt
-
-#Supervisor Config
-RUN mkdir -p /var/log/supervisor
+FROM python:2.7-slim
+ENV DEBIAN_FRONTEND noninteractive
+ENV ETCDCTL_VERSION v2.3.1
+ENV CONFD_VERSION 0.12.0-alpha3
+
+# Native packages, python global deps, gettext, supervisor, dumb-init, etcd
+RUN apt-get update --fix-missing \
+  && apt-get install -y \
+     gettext \
+     wget \
+     curl \
+     openssl \
+
+  # Upgrade pip
+  && pip install --upgrade pip \
+
+  # Confd
+  && curl -L https://github.com/kelseyhightower/confd/releases/download/v$CONFD_VERSION/confd-${CONFD_VERSION}-linux-amd64 -o /usr/local/bin/confd \
+  && chmod 555 /usr/local/bin/confd \
+
+  # Etcd
+  && curl -L https://github.com/coreos/etcd/releases/download/$ETCDCTL_VERSION/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz -o /tmp/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz \
+  && cd /tmp && gzip -dc etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz | tar -xof -  \
+  && cp -f /tmp/etcd-$ETCDCTL_VERSION-linux-amd64/etcdctl /usr/local/bin \
+
+  # Python Global Deps
+  && pip install supervisor==3.2.3 supervisor-stdout \
+
+  # Supervisor Directories
+  && mkdir -p /var/log/supervisor  \
+  && mkdir -p /etc/supervisor/conf.d  \
+  && ln -sf /etc/supervisor/supervisord.conf /etc/supervisord.conf \
+
+  # Dumb Init
+  && wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.0.0/dumb-init_1.0.0_amd64 \
+  && chmod +x /usr/bin/dumb-init \
+
+  # SSH Key for fleet
+  && mkdir /root/.ssh  \
+  && chmod  500 /root/.ssh  \
+  && chown -R root:root /root/.ssh \
+
+  # Cleanup
+  && apt-get clean \
+  && rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /tmp/* /root/.cache/*
+
+# App dependencies
+ADD requirements.txt /opt/cluster-deployer/requirements.txt
+RUN apt-get update --fix-missing \
+  # Install dev packages for build (Remove them in the end)
+  && apt-get install -y gcc libffi-dev libssl-dev \
+  && pip install -r /opt/cluster-deployer/requirements.txt \
+  # Cleanup
+  && apt-get -y remove gcc libffi-dev libssl-dev \
+  && apt-get clean \
+  && rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /tmp/* /root/.cache/*
+
+
+# Supervisor Scripts
 ADD bin/supervisord-wrapper.sh /usr/sbin/supervisord-wrapper.sh
-RUN chmod +x /usr/sbin/supervisord-wrapper.sh && \
-    ln -sf /etc/supervisor/supervisord.conf /etc/supervisord.conf
+RUN chmod +x /usr/sbin/supervisord-wrapper.sh
 
-#Confd Defaults
+# Confd Defaults
 ADD bin/confd-wrapper.sh /usr/sbin/confd-wrapper.sh
 RUN chmod +x /usr/sbin/confd-wrapper.sh
 
-#SSH Keys
+# SSH Keys
 ADD bin/decrypt-ssh-keys.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/decrypt-ssh-keys.sh
 
-#Etc Config
+# Etc Config
 ADD etc /etc
 
 ADD . /opt/cluster-deployer
-RUN pip install -r /opt/cluster-deployer/requirements.txt
 
 EXPOSE 9000
 
 WORKDIR /opt/cluster-deployer
 
-ENTRYPOINT ["/usr/sbin/supervisord-wrapper.sh"]
+CMD ["/usr/bin/dumb-init", "/usr/sbin/supervisord-wrapper.sh"]
diff --git a/bin/confd-wrapper.sh b/bin/confd-wrapper.sh
@@ -1,8 +1,6 @@
 #!/bin/bash -le
 
-export ETCD_URL="$ETCD_HOST:$ETCD_PORT"
-export ETCDCTL="etcdctl --peers $ETCD_URL"
+sed -i -e "s|http[:]//172.17.42.1[:]4001|$ETCD_URL|g" -e "s|/totem|$ETCD_TOTEM_BASE|g" /etc/confd/confd.toml
 
-
-sed -i -e "s/172.17.42.1[:]4001/$ETCD_URL/g" -e "s|/totem|$ETCD_TOTEM_BASE|g" /etc/confd/confd.toml
+$ETCDCTL ls $ETCD_TOTEM_BASE/ssh  || $ETCDCTL mkdir $ETCD_TOTEM_BASE/ssh
 confd
diff --git a/bin/supervisord-wrapper.sh b/bin/supervisord-wrapper.sh
@@ -1,60 +1,75 @@
 #!/bin/bash -le
 
-HOST_IP="${HOST_IP:-$(/sbin/ip route|awk '/default/ { print $3 }')}"
-
-cat <<END>> /etc/profile.d/cluster-deployer-env.sh
-export ETCD_HOST='${ETCD_HOST:-$HOST_IP}'
-export ETCD_PORT='${ETCD_PORT:-4001}'
-export ETCD_TOTEM_BASE='${ETCD_TOTEM_BASE:-/totem}'
-export ETCD_YODA_BASE='${ETCD_YODA_BASE:-/yoda}'
-export CELERY_GEVENT_EXECUTORS='${CELERY_GEVENT_EXECUTORS:-1}'
-export CELERY_GEVENT_CONCURRENCY='${CELERY_GEVENT_CONCURRENCY:-50}'
-export CELERY_PREFORK_CONCURRENCY='${CELERY_PREFORK_CONCURRENCY:-2}'
-export API_EXECUTORS='${API_EXECUTORS:-2}'
-export C_FORCE_ROOT='${C_FORCE_ROOT:-true}'
-export SSH_HOST_KEY='${SSH_HOST_KEY:-/root/.ssh/id_rsa}'
-export AMQP_HOST='${AMQP_HOST:-$HOST_IP}'
-export AMQP_PORT='${AMQP_PORT:-5672}'
-export AMQP_USERNAME='${AMQP_USERNAME:-guest}'
-export AMQP_PASSWORD='${AMQP_PASSWORD:-guest}'
-export MONGODB_USERNAME='${MONGODB_USERNAME:-}'
-export MONGODB_PASSWORD='${MONGODB_PASSWORD:-}'
-export MONGODB_HOST='${MONGODB_HOST:-$HOST_IP}'
-export MONGODB_PORT='${MONGODB_PORT:-27017}'
-export MONGODB_DB='${MONGODB_DB}'
-export MONGODB_AUTH_DB='${MONGODB_AUTH_DB}'
-export BROKER_URL='${BROKER_URL}'
-export SSH_PASSPHRASE='${SSH_PASSPHRASE}'
-export GITHUB_TOKEN='${GITHUB_TOKEN}'
-export FLEET_HOST='${FLEET_HOST:-$HOST_IP}'
-export CLUSTER_NAME='${CLUSTER_NAME:-local}'
-export TOTEM_ENV='${TOTEM_ENV:-local}'
-export QUAY_ORGANIZATION='${QUAY_ORGANIZATION:-totem}'
-export QUAY_PREFIX='${QUAY_PREFIX:-totem-}'
+export HOST_IP="${HOST_IP:-$(/sbin/ip route|awk '/default/ { print $3 }')}"
+export ETCD_HOST="${ETCD_HOST:-HOST_IP}"
+export ETCD_PORT="${ETCD_PORT:-4001}"
+export ETCD_URL="${ETCD_URL:-http://$ETCD_HOST:$ETCD_PORT}"
+export ETCDCTL="${ETCDCTL:-etcdctl --peers $ETCD_URL}"
+export ETCD_TOTEM_BASE="${ETCD_TOTEM_BASE}"
+export ETCD_YODA_BASE="${ETCD_YODA_BASE:-/yoda}"
+export CELERY_GEVENT_EXECUTORS="${CELERY_GEVENT_EXECUTORS:-1}"
+export CELERY_GEVENT_CONCURRENCY="${CELERY_GEVENT_CONCURRENCY:-50}"
+export CELERY_PREFORK_CONCURRENCY="${CELERY_PREFORK_CONCURRENCY:-2}"
+export API_EXECUTORS="${API_EXECUTORS:-2}"
+export C_FORCE_ROOT="${C_FORCE_ROOT:-true}"
+export SSH_HOST_KEY="${SSH_HOST_KEY:-/root/.ssh/id_rsa}"
+export AMQP_HOST="${AMQP_HOST:-$HOST_IP}"
+export AMQP_PORT="${AMQP_PORT:-5672}"
+export AMQP_USERNAME="${AMQP_USERNAME:-guest}"
+export AMQP_PASSWORD="${AMQP_PASSWORD:-guest}"
+export MONGODB_USERNAME="${MONGODB_USERNAME:-}"
+export MONGODB_PASSWORD="${MONGODB_PASSWORD:-}"
+export MONGODB_SERVERS="${MONGODB_SERVERS:-}"
+export MONGODB_HOST="${MONGODB_HOST:-$HOST_IP}"
+export MONGODB_PORT="${MONGODB_PORT:-27017}"
+export MONGODB_DB="${MONGODB_DB}"
+export MONGODB_AUTH_DB="${MONGODB_AUTH_DB}"
+export BROKER_URL="${BROKER_URL}"
+export SSH_PASSPHRASE="${SSH_PASSPHRASE}"
+export GITHUB_TOKEN="${GITHUB_TOKEN}"
+export FLEET_HOST="${FLEET_HOST:-$HOST_IP}"
+export CLUSTER_NAME="${CLUSTER_NAME:-local}"
+export TOTEM_ENV="${TOTEM_ENV:-local}"
+export QUAY_ORGANIZATION="${QUAY_ORGANIZATION:-totem}"
+export QUAY_PREFIX="${QUAY_PREFIX:-totem-}"
 export C_FORCE_ROOT=true
-export ENCRYPTION_PASSPHRASE='${ENCRYPTION_PASSPHRASE:-changeit}'
-export ENCRYPTION_S3_BUCKET='${ENCRYPTION_S3_BUCKET:-not-set}'
-export ENCRYPTION_STORE='${ENCRYPTION_PROVIDER:-s3}'
-export HIPCHAT_TOKEN='${HIPCHAT_TOKEN}'
-export HIPCHAT_ENABLED='${HIPCHAT_ENABLED:-false}'
-export HIPCHAT_TOKEN='${HIPCHAT_TOKEN}'
-export HIPCHAT_ROOM='${HIPCHAT_ROOM:-not-set}'
-export GITHUB_NOTIFICATION_ENABLED='${GITHUB_NOTIFICATION_ENABLED:-false}'
-export BASE_URL='${BASE_URL:-http://$HOST_IP:9000}'
-export LOG_IDENTIFIER='${LOG_IDENTIFIER:-cluster-deployer}'
-export LOG_ROOT_LEVEL='${LOG_ROOT_LEVEL}'
-END
+export ENCRYPTION_PASSPHRASE="${ENCRYPTION_PASSPHRASE:-changeit}"
+export ENCRYPTION_S3_BUCKET="${ENCRYPTION_S3_BUCKET:-not-set}"
+export ENCRYPTION_STORE="${ENCRYPTION_PROVIDER:-s3}"
+export HIPCHAT_TOKEN="${HIPCHAT_TOKEN}"
+export HIPCHAT_ENABLED="${HIPCHAT_ENABLED:-false}"
+export HIPCHAT_TOKEN="${HIPCHAT_TOKEN}"
+export HIPCHAT_ROOM="${HIPCHAT_ROOM:-not-set}"
+export GITHUB_NOTIFICATION_ENABLED="${GITHUB_NOTIFICATION_ENABLED:-false}"
+export BASE_URL="${BASE_URL:-http://$HOST_IP:9000}"
+export LOG_IDENTIFIER="${LOG_IDENTIFIER:-cluster-deployer}"
+export LOG_ROOT_LEVEL="${LOG_ROOT_LEVEL}"
 
-echo "Registering shutdown hook prior to shutdown"
-function shutdown() {
-    set +e;
-    echo Stopping supervisor;
-    kill -s SIGTERM "$(cat /var/run/supervisord.pid)";
-    exit 0
-}
-trap 'shutdown' EXIT
+until $ETCDCTL cluster-health; do
+  >&2 echo "Etcdctl cluster not healthy - sleeping"
+  sleep 5
+done
 
-/bin/bash -le -c " envsubst  < /etc/supervisor/conf.d/supervisord.conf.template  > /etc/supervisor/conf.d/supervisord.conf; \
-                    /usr/local/bin/supervisord -c /etc/supervisor/supervisord.conf"
+# Cleanup for local testing
+if ls celerybeat* 2>/dev/null; then
+  rm celerybeat*
+fi
 
+if [ "$DISCOVER_RABBITMQ" == "true" ]; then
+  AMQP_HOST="$($ETCDCTL ls ${ETCD_TOTEM_BASE}/rabbitmq/nodes | xargs -L1  etcdctl get | tr '\n' ',')"
+  if [ ! -z "$AMQP_HOST" ]; then
+    echo "No rabbitmq nodes could be discovered. Exiting cluster-deployer"
+    exit 1
+  fi
+fi
 
+if [ "$DISCOVER_MONGO" == "true" ]; then
+  MONGODB_SERVERS="$($ETCDCTL ls ${ETCD_TOTEM_BASE}/mongodb/nodes | xargs -L1  etcdctl get | tr '\n' ',')"
+  if [ ! -z "$MONGODB_SERVERS" ]; then
+    echo "No mongodb nodes could be discovered. Exiting cluster-deployer"
+    exit 1
+  fi
+fi
+
+/bin/bash -le -c " envsubst  < /etc/supervisor/conf.d/supervisord.conf.template  > /etc/supervisor/conf.d/supervisord.conf; \
+                    /usr/local/bin/supervisord -c /etc/supervisor/supervisord.conf"
diff --git a/conf/appconfig.py b/conf/appconfig.py
@@ -279,12 +279,15 @@
 MONGODB_PASSWORD = os.getenv('MONGODB_PASSWORD', '')
 MONGODB_HOST = os.getenv('MONGODB_HOST', '127.0.0.1')
 MONGODB_PORT = int(os.getenv('MONGODB_PORT', '27017'))
+MONGODB_SERVERS = os.getenv('MONGODB_SERVERS', '{}:{}'.format(
+    MONGODB_HOST, MONGODB_PORT))
 MONGODB_DB = os.getenv('MONGODB_DB') or 'totem-{}'.format(TOTEM_ENV)
 MONGODB_AUTH_DB = os.getenv('MONGODB_AUTH_DB') or 'admin'
 MONGODB_AUTH = '{0}:{1}@'.format(MONGODB_USERNAME, MONGODB_PASSWORD) \
     if MONGODB_USERNAME else ''
-MONGODB_DEFAULT_URL = 'mongodb://{0}{1}:{2}/{3}'.format(
-    MONGODB_AUTH, MONGODB_HOST, MONGODB_PORT, MONGODB_AUTH_DB)
+
+MONGODB_DEFAULT_URL = 'mongodb://{0}{1}/{2}'.format(
+    MONGODB_AUTH, MONGODB_SERVERS, MONGODB_AUTH_DB)
 MONGODB_URL = os.getenv('MONGODB_URL') or MONGODB_DEFAULT_URL
 MONGODB_DEPLOYMENT_COLLECTION = os.getenv('MONGODB_DEPLOYMENT_COLLECTION') or \
     'deployments'

diff --git a/conf/celeryconfig.py b/conf/celeryconfig.py
@@ -17,8 +17,11 @@
 AMQP_PASSWORD = os.getenv('AMQP_PASSWORD', 'guest')
 AMQP_HOST = os.getenv('AMQP_HOST', 'localhost')
 AMQP_PORT = int(os.getenv('AMQP_PORT', '5672'))
-DEFAULT_BROKER_URL = 'amqp://%s:%s@%s:%s' % (AMQP_USERNAME, AMQP_PASSWORD,
-                                             AMQP_HOST, AMQP_PORT)
+DEFAULT_BROKER_URL = [
+    'amqp://{}:{}@{}:{}'.format(
+        AMQP_USERNAME, AMQP_PASSWORD, host.strip(), AMQP_PORT
+    ) for host in AMQP_HOST.split(',') if host.strip()
+]
 BROKER_URL = os.getenv('BROKER_URL') or DEFAULT_BROKER_URL
 
 BROKER_CONNECTION_TIMEOUT = int(os.getenv('BROKER_CONNECTION_TIMEOUT', '20'))
@@ -59,6 +62,7 @@
 }
 
 # Backend Settings
+print('Mongo URL %s', MONGODB_URL)
 CELERY_RESULT_BACKEND = MONGODB_URL
 CELERY_MONGODB_BACKEND_SETTINGS = {
     'database': MONGODB_DB,
@@ -112,12 +116,12 @@
     },
     'deployer.tasks.deployment.sync_promoted_units': {
         'task': 'deployer.tasks.deployment.sync_promoted_units',
-        'schedule': crontab(minute='*/5'),
+        'schedule': crontab(minute='*/15'),
         'args': ()
     },
     'deployer.tasks.deployment.sync_promoted_upstreams': {
         'task': 'deployer.tasks.deployment.sync_promoted_upstreams',
-        'schedule': crontab(minute='*/2'),
+        'schedule': crontab(minute='*/15'),
         'args': ()
     }
 }

diff --git a/deployer/__init__.py b/deployer/__init__.py
@@ -3,7 +3,7 @@
 from celery.signals import setup_logging
 
 
-__version__ = '0.4.8'
+__version__ = '0.5.0'
 __author__ = 'sukrit'
 
 deployer.logger.init_logging()