Support for mongo store for cluster deployer

.travis.yml

@@ -4,6 +4,9 @@ python:
 
 sudo: false
 
+services:
+  - mongodb
+
 # command to install dependencies
 install:
   - "pip install -r requirements.txt"

conf/appconfig.py

@@ -3,7 +3,7 @@
 # Logging configuration
 LOG_FORMAT = '%(asctime)s [%(name)s] %(levelname)s %(message)s'
 LOG_DATE = '%Y-%m-%d %I:%M:%S %p'
-LOG_ROOT_LEVEL = os.getenv('LOG_ROOT_LEVEL', 'DEBUG').upper()
+LOG_ROOT_LEVEL = os.getenv('LOG_ROOT_LEVEL', 'INFO').upper()
 LOG_IDENTIFIER = os.getenv('LOG_IDENTIFIER', 'cluster-deployer')
 
 
@@ -18,11 +18,15 @@
 DEPLOYMENT_MODE_AB = 'a/b'
 DEPLOYMENT_MODE_CUSTOM = 'custom'
 
+DEPLOYMENT_STATE_NEW = 'NEW'
 DEPLOYMENT_STATE_STARTED = 'STARTED'
 DEPLOYMENT_STATE_PROMOTED = 'PROMOTED'
 DEPLOYMENT_STATE_FAILED = 'FAILED'
 DEPLOYMENT_STATE_DECOMMISSIONED = 'DECOMMISSIONED'
 
+RUNNING_DEPLOYMENT_STATES = [DEPLOYMENT_STATE_NEW, DEPLOYMENT_STATE_STARTED,
+                             DEPLOYMENT_STATE_PROMOTED]
+
 BOOLEAN_TRUE_VALUES = {"true", "yes", "y", "1", "on"}
 
 API_PORT = int(os.getenv('API_PORT', '9000'))
@@ -80,7 +84,7 @@
                 'name': 'default-app',
             },
             'yoda-register': {
-                'name': 'yoda-ec2-register'
+                'name': 'yoda-register'
             }
         }
     },
@@ -161,16 +165,14 @@
     'DEFAULT_RETRY_DELAY': 10,
     'SSH_RETRY_DELAY': 10,
     'SSH_RETRIES': 10,
-    'CHECK_RUNNING_RETRIES': 60,
-    'CHECK_RUNNING_RETRY_DELAY': 10,
+    'CHECK_RUNNING_RETRIES': 30,
+    'CHECK_RUNNING_RETRY_DELAY': 30,
     'CHECK_DISCOVERY_RETRIES': 20,
     'CHECK_DISCOVERY_RETRY_DELAY': 30,
     'LOCK_RETRIES': 64,
     'LOCK_RETRY_DELAY': 60,
-    'DEPLOYMENT_WAIT_RETRIES': 40,
-    'DEPLOYMENT_WAIT_RETRY_DELAY': 30,
-    'CHECK_DEPLOYMENT_RETRY_DELAY': 20,
-    'CHECK_DEPLOYMENT_RETRIES': 30,
+    'DEPLOYMENT_WAIT_RETRIES': 60,
+    'DEPLOYMENT_WAIT_RETRY_DELAY': 60,
     'CHECK_NODE_RETRY_DELAY': 10,
 }
 
@@ -243,14 +245,25 @@
 DOC_TYPE_DEPLOYMENTS = 'deployments'
 DOC_TYPE_EVENTS = 'events'
 
+# Storage
+DEFAULT_STORE_NAME = 'mongo'
 # Mongo Settings
 MONGODB_USERNAME = os.getenv('MONODB_USERNAME', '')
 MONGODB_PASSWORD = os.getenv('MONODB_PASSWORD', '')
-MONGODB_HOST = os.getenv('MONGODB_HOST', '172.17.42.1')
+MONGODB_HOST = os.getenv('MONGODB_HOST', '127.0.0.1')
 MONGODB_PORT = int(os.getenv('MONGODB_PORT', '27017'))
 MONGODB_DB = os.getenv('MONGODB_DB', 'totem')
 MONGODB_AUTH = '{0}:{1}@'.format(MONGODB_USERNAME, MONGODB_PASSWORD) \
     if MONGODB_USERNAME else ''
 MONGODB_DEFAULT_URL = 'mongodb://{0}{1}:{2}/{3}'.format(
     MONGODB_AUTH, MONGODB_HOST, MONGODB_PORT, MONGODB_DB)
 MONGODB_URL = os.getenv('MONGODB_URL') or MONGODB_DEFAULT_URL
+MONGODB_DEPLOYMENT_COLLECTION = os.getenv('MONGODB_DEPLOYMENT_COLLECTION') or \
+    'deployments'
+MONGODB_EVENT_COLLECTION = os.getenv('MONGODB_EVENT_COLLECTION') or \
+    'events'
+
+# Number of seconds after a non running deployment will expire
+DEFAULT_DEPLOYMENT_EXPIRY_SECONDS = 4 * 7 * 24 * 3600  # 4 weeks
+DEPLOYMENT_EXPIRY_SECONDS = int(
+    os.getenv('DEPLOYMENT_EXPIRY_SECONDS', DEFAULT_DEPLOYMENT_EXPIRY_SECONDS))

conf/celeryconfig.py

@@ -58,12 +58,11 @@
 CELERY_RESULT_BACKEND = MONGODB_URL
 CELERY_MONGODB_BACKEND_SETTINGS = {
     'database': MONGODB_DB,
-    'taskmeta_collection': 'deployer-task-results-%s' % CLUSTER_NAME,
+    'taskmeta_collection': 'deployer-task-results',
 }
-CELERY_RESULT_EXCHANGE = 'cluster-deployer-%s-results' % CLUSTER_NAME
+CELERY_RESULT_EXCHANGE = 'cluster-deployer-results'
 CELERY_IMPORTS = ('deployer.tasks', 'deployer.tasks.deployment',
-                  'deployer.tasks.common', 'deployer.tasks.proxy',
-                  'celery.task')
+                  'deployer.tasks.common', 'celery.task')
 CELERY_ACCEPT_CONTENT = ['json', 'pickle']
 CELERY_TASK_SERIALIZER = 'pickle'
 CELERY_RESULT_SERIALIZER = 'pickle'

deployer/services/health.py

@@ -4,6 +4,7 @@
 from conf.appconfig import HEALTH_OK, HEALTH_FAILED, TOTEM_ETCD_SETTINGS, \
     SEARCH_SETTINGS
 from deployer.elasticsearch import get_search_client
+from deployer.services.storage.factory import get_store
 from deployer.tasks.common import ping
 from deployer.util import timeout
 
@@ -61,6 +62,15 @@ def _check_etcd():
     }
 
 
+@timeout(HEALTH_TIMEOUT_SECONDS)
+@_check
+def _check_store():
+    """
+    Checks health of default store
+    """
+    return get_store().health()
+
+
 @timeout(HEALTH_TIMEOUT_SECONDS)
 @_check
 def _check_celery():
@@ -83,6 +93,7 @@ def get_health(check_celery=True):
 
     health_status = {
         'etcd': _check_etcd(),
+        'store': _check_store()
     }
     if check_celery:
         health_status['celery'] = _check_celery()