Merge pull request #26 from totem/develop

0.3 Release

.travis.yml

@@ -4,6 +4,9 @@ python:
 
 sudo: false
 
+services:
+  - mongodb
+
 # command to install dependencies
 install:
   - "pip install -r requirements.txt"

Dockerfile

@@ -1,4 +1,4 @@
-FROM totem/python-base:2.7-trusty-b2
+FROM totem/python-base:2.7-trusty-b3
 
 ENV DEBIAN_FRONTEND noninteractive
 
@@ -25,7 +25,7 @@ RUN curl -L https://github.com/coreos/etcd/releases/download/$ETCDCTL_VERSION/et
     rm -rf /tmp/etcd-$ETCDCTL_VERSION-linux-amd64
 
 # Supervisor and App dependencies
-RUN pip install supervisor==3.1.2
+RUN pip install supervisor==3.1.2 supervisor-stdout
 ADD requirements.txt /opt/requirements.txt
 RUN pip install -r /opt/requirements.txt
 

README.md

@@ -98,8 +98,12 @@ sudo docker run -it --rm -h cluster-deployer-${USER} --name cluster-deployer -v
 | CLUSTER_NAME | Name of the cluster where orchestrator is deployed | local | local |
 | TOTEM_ENV | Name of totem environment (e.g. production, local, development) | local | local |
 | LOG_IDENTIFIER | Program name/tag used for syslog | N/A | yoda-proxy |
-
-
+| MONGODB_HOST | Mongo db host | 127.0.0.1 | HOST_IP |
+| MONGODB_PORT | Mongo db port | 27017 | 27017 |
+| MONGODB_AUTH_DB | Mongo db authentication database | admin | admin |
+| MONGODB_USERNAME | Mongo db username | | |
+| MONGODB_PASSWORD | Mongo db password | | |
+| MONGODB_DB | Mongo db database | totem-{TOTEM_ENV} | totem-{TOTEM_ENV} |
 
 ## Coding Standards and Guidelines
 

bin/supervisord-wrapper.sh

@@ -1,7 +1,9 @@
 #!/bin/bash -le
 
+HOST_IP="${HOST_IP:-$(/sbin/ip route|awk '/default/ { print $3 }')}"
+
 cat <<END>> /etc/profile.d/cluster-deployer-env.sh
-export ETCD_HOST='${ETCD_HOST:-172.17.42.1}'
+export ETCD_HOST='${ETCD_HOST:-$HOST_IP}'
 export ETCD_PORT='${ETCD_PORT:-4001}'
 export ETCD_TOTEM_BASE='${ETCD_TOTEM_BASE:-/totem}'
 export ETCD_YODA_BASE='${ETCD_YODA_BASE:-/yoda}'
@@ -11,21 +13,24 @@ export CELERY_PREFORK_CONCURRENCY='${CELERY_PREFORK_CONCURRENCY:-2}'
 export API_EXECUTORS='${API_EXECUTORS:-2}'
 export C_FORCE_ROOT='${C_FORCE_ROOT:-true}'
 export SSH_HOST_KEY='${SSH_HOST_KEY:-/root/.ssh/id_rsa}'
-export AMQP_HOST='${AMQP_HOST:-172.17.42.1}'
+export AMQP_HOST='${AMQP_HOST:-$HOST_IP}'
 export AMQP_PORT='${AMQP_PORT:-5672}'
 export AMQP_USERNAME='${AMQP_USERNAME:-guest}'
 export AMQP_PASSWORD='${AMQP_PASSWORD:-guest}'
+export MONGODB_USERNAME='${MONGODB_USERNAME:-}'
+export MONGODB_PASSWORD='${MONGODB_PASSWORD:-}'
+export MONGODB_HOST='${MONGODB_HOST:-$HOST_IP}'
+export MONGODB_PORT='${MONGODB_PORT:-27017}'
+export MONGODB_DB='${MONGODB_DB}'
+export MONGODB_AUTH_DB='${MONGODB_AUTH_DB}'
 export BROKER_URL='${BROKER_URL}'
 export SSH_PASSPHRASE='${SSH_PASSPHRASE}'
 export GITHUB_TOKEN='${GITHUB_TOKEN}'
-export ELASTICSEARCH_HOST='${ELASTICSEARCH_HOST:-172.17.42.1}'
-export ELASTICSEARCH_PORT='${ELASTICSEARCH_PORT:-9200}'
-export FLEET_HOST='${FLEET_HOST:-172.17.42.1}'
+export FLEET_HOST='${FLEET_HOST:-$HOST_IP}'
 export CLUSTER_NAME='${CLUSTER_NAME:-local}'
 export TOTEM_ENV='${TOTEM_ENV:-local}'
 export QUAY_ORGANIZATION='${QUAY_ORGANIZATION:-totem}'
 export QUAY_PREFIX='${QUAY_PREFIX:-totem-}'
-export SEARCH_ENABLED=${SEARCH_ENABLED:-false}
 export C_FORCE_ROOT=true
 export ENCRYPTION_PASSPHRASE='${ENCRYPTION_PASSPHRASE:-changeit}'
 export ENCRYPTION_S3_BUCKET='${ENCRYPTION_S3_BUCKET:-not-set}'
@@ -35,7 +40,7 @@ export HIPCHAT_ENABLED='${HIPCHAT_ENABLED:-false}'
 export HIPCHAT_TOKEN='${HIPCHAT_TOKEN}'
 export HIPCHAT_ROOM='${HIPCHAT_ROOM:-not-set}'
 export GITHUB_NOTIFICATION_ENABLED='${GITHUB_NOTIFICATION_ENABLED:-false}'
-export BASE_URL='${BASE_URL:-http://172.17.42.1:9000}'
+export BASE_URL='${BASE_URL:-http://$HOST_IP:9000}'
 export LOG_IDENTIFIER='${LOG_IDENTIFIER:-cluster-deployer}'
 END
 

conf/appconfig.py

@@ -18,11 +18,15 @@
 DEPLOYMENT_MODE_AB = 'a/b'
 DEPLOYMENT_MODE_CUSTOM = 'custom'
 
+DEPLOYMENT_STATE_NEW = 'NEW'
 DEPLOYMENT_STATE_STARTED = 'STARTED'
 DEPLOYMENT_STATE_PROMOTED = 'PROMOTED'
 DEPLOYMENT_STATE_FAILED = 'FAILED'
 DEPLOYMENT_STATE_DECOMMISSIONED = 'DECOMMISSIONED'
 
+RUNNING_DEPLOYMENT_STATES = [DEPLOYMENT_STATE_NEW, DEPLOYMENT_STATE_STARTED,
+                             DEPLOYMENT_STATE_PROMOTED]
+
 BOOLEAN_TRUE_VALUES = {"true", "yes", "y", "1", "on"}
 
 API_PORT = int(os.getenv('API_PORT', '9000'))
@@ -38,6 +42,10 @@
 DEFAULT_GITHUB_TOKEN = os.getenv('GITHUB_TOKEN', '')
 BASE_URL = os.getenv('BASE_URL', 'http://localhost:{0}'.format(API_PORT))
 
+TOTEM_ENV = os.getenv('TOTEM_ENV', 'local')
+CLUSTER_NAME = os.getenv('CLUSTER_NAME', TOTEM_ENV)
+GIT_COMMIT = os.getenv('GIT_COMMIT', 'latest')
+
 NOTIFICATIONS_DEFAULTS = {
     'hipchat': {
         'enabled': os.getenv('HIPCHAT_ENABLED', 'false').strip()
@@ -80,7 +88,7 @@
                 'name': 'default-app',
             },
             'yoda-register': {
-                'name': 'yoda-ec2-register'
+                'name': 'yoda-register'
             }
         }
     },
@@ -155,44 +163,35 @@
 
 DEFAULT_LOCK_TTL = 3600
 
+LOCK_JOB_TTL = 120
+LOCK_JOB_BASE = '/cluster-deployer/locks/jobs'
+LOCK_JOB_SYNC_PROMOTED_UPSTREAMS = 'sync-promoted-upstreams'
+LOCK_JOB_SYNC_PROMOTED_UNITS = 'sync-promoted-units'
+
 TASK_SETTINGS = {
     'DEFAULT_GET_TIMEOUT': 600,
     'DEFAULT_RETRIES': 5,
     'DEFAULT_RETRY_DELAY': 10,
     'SSH_RETRY_DELAY': 10,
     'SSH_RETRIES': 10,
-    'CHECK_RUNNING_RETRIES': 60,
-    'CHECK_RUNNING_RETRY_DELAY': 10,
+    'CHECK_RUNNING_RETRIES': 30,
+    'CHECK_RUNNING_RETRY_DELAY': 30,
     'CHECK_DISCOVERY_RETRIES': 20,
     'CHECK_DISCOVERY_RETRY_DELAY': 30,
     'LOCK_RETRIES': 64,
     'LOCK_RETRY_DELAY': 60,
-    'DEPLOYMENT_WAIT_RETRIES': 40,
-    'DEPLOYMENT_WAIT_RETRY_DELAY': 30,
-    'CHECK_DEPLOYMENT_RETRY_DELAY': 20,
-    'CHECK_DEPLOYMENT_RETRIES': 30,
+    'DEPLOYMENT_WAIT_RETRIES': 60,
+    'DEPLOYMENT_WAIT_RETRY_DELAY': 60,
     'CHECK_NODE_RETRY_DELAY': 10,
 }
 
 TOTEM_ETCD_SETTINGS = {
     'base': os.getenv('ETCD_TOTEM_BASE', '/totem'),
-    'host': os.getenv('ETCD_HOST', '172.17.42.1'),
+    'host': os.getenv('ETCD_HOST', '127.0.0.1'),
     'port': int(os.getenv('ETCD_PORT', '4001')),
     'yoda_base': os.getenv('ETCD_YODA_BASE', '/yoda'),
 }
 
-TOTEM_ENV = os.getenv('TOTEM_ENV', 'local')
-CLUSTER_NAME = os.getenv('CLUSTER_NAME', TOTEM_ENV)
-SEARCH_INDEX = os.getenv('SEARCH_INDEX', 'totem-{0}'.format(TOTEM_ENV))
-
-SEARCH_SETTINGS = {
-    'enabled': os.getenv('SEARCH_ENABLED', 'false').strip().lower() in
-    BOOLEAN_TRUE_VALUES,
-    'host': os.getenv('ELASTICSEARCH_HOST', '172.17.42.1'),
-    'port': os.getenv('ELASTICSEARCH_PORT', '9200'),
-    'default-index': SEARCH_INDEX
-}
-
 CORS_SETTINGS = {
     'enabled': os.getenv('CORS_ENABLED', 'true').strip().lower() in
     BOOLEAN_TRUE_VALUES,
@@ -223,6 +222,7 @@
     'application/vnd.deployer.app.version.delete.v1+json'
 MIME_APP_DELETE_V1 = 'application/vnd.deployer.app.delete.v1+json'
 MIME_HEALTH_V1 = 'application/vnd.deployer.health.v1+json'
+MIME_RECOVERY_V1 = 'application/vnd.deployer.recovery.v1+json'
 
 SCHEMA_TASK_V1 = 'task-v1'
 SCHEMA_ROOT_V1 = 'root-v1'
@@ -232,13 +232,34 @@
 SCHEMA_APP_VERSION_LIST_V1 = 'app-version-list-v1'
 SCHEMA_APP_VERSION_UNIT_LIST_V1 = 'app-version-unit-list-v1'
 SCHEMA_HEALTH_V1 = 'health-v1'
+SCHEMA_RECOVERY_V1 = 'recovery-v1'
 
 API_MAX_PAGE_SIZE = 1000
 API_DEFAULT_PAGE_SIZE = 10
 
 HEALTH_OK = 'ok'
 HEALTH_FAILED = 'failed'
 
-# Elasticsearch doc types
-DOC_TYPE_DEPLOYMENTS = 'deployments'
-DOC_TYPE_EVENTS = 'events'
+# Storage
+DEFAULT_STORE_NAME = 'mongo'
+# Mongo Settings
+MONGODB_USERNAME = os.getenv('MONGODB_USERNAME', '')
+MONGODB_PASSWORD = os.getenv('MONGODB_PASSWORD', '')
+MONGODB_HOST = os.getenv('MONGODB_HOST', '127.0.0.1')
+MONGODB_PORT = int(os.getenv('MONGODB_PORT', '27017'))
+MONGODB_DB = os.getenv('MONGODB_DB') or 'totem-{}'.format(TOTEM_ENV)
+MONGODB_AUTH_DB = os.getenv('MONGODB_AUTH_DB') or 'admin'
+MONGODB_AUTH = '{0}:{1}@'.format(MONGODB_USERNAME, MONGODB_PASSWORD) \
+    if MONGODB_USERNAME else ''
+MONGODB_DEFAULT_URL = 'mongodb://{0}{1}:{2}/{3}'.format(
+    MONGODB_AUTH, MONGODB_HOST, MONGODB_PORT, MONGODB_AUTH_DB)
+MONGODB_URL = os.getenv('MONGODB_URL') or MONGODB_DEFAULT_URL
+MONGODB_DEPLOYMENT_COLLECTION = os.getenv('MONGODB_DEPLOYMENT_COLLECTION') or \
+    'deployments'
+MONGODB_EVENT_COLLECTION = os.getenv('MONGODB_EVENT_COLLECTION') or \
+    'events'
+
+# Number of seconds after a non running deployment will expire
+DEFAULT_DEPLOYMENT_EXPIRY_SECONDS = 4 * 7 * 24 * 3600  # 4 weeks
+DEPLOYMENT_EXPIRY_SECONDS = int(
+    os.getenv('DEPLOYMENT_EXPIRY_SECONDS', DEFAULT_DEPLOYMENT_EXPIRY_SECONDS))

conf/celeryconfig.py

@@ -4,12 +4,14 @@
 
 from celery.schedules import crontab
 from kombu import Queue
+from conf.appconfig import MONGODB_DB, MONGODB_URL
 
 TOTEM_ENV = os.getenv('TOTEM_ENV', 'local')
 CLUSTER_NAME = os.getenv('CLUSTER_NAME', TOTEM_ENV)
 
 MESSAGES_TTL = 7200 * 1000
 
+
 # Broker and Queue Settings
 AMQP_USERNAME = os.getenv('AMQP_USERNAME', 'guest')
 AMQP_PASSWORD = os.getenv('AMQP_PASSWORD', 'guest')
@@ -49,14 +51,21 @@
     },
     'deployer.tasks.deployment._fleet_check_running': {
         'routing_key': 'prefork',
+    },
+    'deployer.tasks.deployment.sync_units_task': {
+        'routing_key': 'prefork',
     }
 }
 
-CELERY_RESULT_BACKEND = 'amqp'
-CELERY_RESULT_EXCHANGE = 'cluster-deployer-%s-results' % CLUSTER_NAME
+# Backend Settings
+CELERY_RESULT_BACKEND = MONGODB_URL
+CELERY_MONGODB_BACKEND_SETTINGS = {
+    'database': MONGODB_DB,
+    'taskmeta_collection': 'deployer-task-results',
+}
+CELERY_RESULT_EXCHANGE = 'cluster-deployer-results'
 CELERY_IMPORTS = ('deployer.tasks', 'deployer.tasks.deployment',
-                  'deployer.tasks.common', 'deployer.tasks.proxy',
-                  'celery.task')
+                  'deployer.tasks.common', 'celery.task')
 CELERY_ACCEPT_CONTENT = ['json', 'pickle']
 CELERY_TASK_SERIALIZER = 'pickle'
 CELERY_RESULT_SERIALIZER = 'pickle'
@@ -95,6 +104,16 @@
         'task': 'deployer.tasks.backend_cleanup',
         'schedule': crontab(hour="*/2", minute=0),
         'args': (),
+    },
+    'deployer.tasks.deployment.sync_promoted_units': {
+        'task': 'deployer.tasks.deployment.sync_promoted_units',
+        'schedule': crontab(minute='*/5'),
+        'args': ()
+    },
+    'deployer.tasks.deployment.sync_promoted_upstreams': {
+        'task': 'deployer.tasks.deployment.sync_promoted_upstreams',
+        'schedule': crontab(minute='*/2'),
+        'args': ()
     }
 }
 

deployer/__init__.py

@@ -3,8 +3,8 @@
 from celery.signals import setup_logging
 
 
-__version__ = '0.2.2'
+__version__ = '0.3'
 __author__ = 'sukrit'
 
-deployer.logger.init_logging('root')
+deployer.logger.init_logging()
 setup_logging.connect(deployer.logger.init_celery_logging)

deployer/celery.py

@@ -1,19 +1,5 @@
 from __future__ import absolute_import
 from celery import Celery
-from celery.signals import celeryd_init
-from deployer import elasticsearch
 
 app = Celery(__name__)
 app.config_from_object('conf.celeryconfig')
-
-
-@celeryd_init.connect
-def configure_search(**kwargs):
-    """
-    Creates the index mapping for elastic search on startup.
-    If mappings already exists, this will be ignored
-
-    :param kwargs:
-    :return:
-    """
-    elasticsearch.create_index_mapping()